From 5c8d504809b7771b547249d22baf927a9b2799ac Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Wed, 16 Aug 2023 17:09:16 +0900 Subject: [PATCH 1/8] sync test case initial scripts --- cicd/sync/.vagrant/bundler/global.sol | 1 + .../client/virtualbox/action_provision | 1 + .../client/virtualbox/action_set_name | 1 + .../machines/client/virtualbox/box_meta | 1 + .../machines/client/virtualbox/creator_uid | 1 + .../.vagrant/machines/client/virtualbox/id | 1 + .../machines/client/virtualbox/index_uuid | 1 + .../machines/client/virtualbox/private_key | 27 ++++++++++ .../machines/client/virtualbox/synced_folders | 1 + .../machines/client/virtualbox/vagrant_cwd | 1 + .../machines/ep1/virtualbox/action_provision | 1 + .../machines/ep1/virtualbox/action_set_name | 1 + .../.vagrant/machines/ep1/virtualbox/box_meta | 1 + .../machines/ep1/virtualbox/creator_uid | 1 + cicd/sync/.vagrant/machines/ep1/virtualbox/id | 1 + .../machines/ep1/virtualbox/index_uuid | 1 + .../machines/ep1/virtualbox/private_key | 27 ++++++++++ .../machines/ep1/virtualbox/synced_folders | 1 + .../machines/ep1/virtualbox/vagrant_cwd | 1 + .../machines/llb1/virtualbox/action_provision | 1 + .../machines/llb1/virtualbox/action_set_name | 1 + .../machines/llb1/virtualbox/box_meta | 1 + .../machines/llb1/virtualbox/creator_uid | 1 + .../sync/.vagrant/machines/llb1/virtualbox/id | 1 + .../machines/llb1/virtualbox/index_uuid | 1 + .../machines/llb1/virtualbox/private_key | 27 ++++++++++ .../machines/llb1/virtualbox/synced_folders | 1 + .../machines/llb1/virtualbox/vagrant_cwd | 1 + .../machines/llb2/virtualbox/action_provision | 1 + .../machines/llb2/virtualbox/action_set_name | 1 + .../machines/llb2/virtualbox/box_meta | 1 + .../machines/llb2/virtualbox/creator_uid | 1 + .../sync/.vagrant/machines/llb2/virtualbox/id | 1 + .../machines/llb2/virtualbox/index_uuid | 1 + .../machines/llb2/virtualbox/private_key | 27 ++++++++++ .../machines/llb2/virtualbox/synced_folders | 1 + .../machines/llb2/virtualbox/vagrant_cwd | 1 + cicd/sync/.vagrant/rgloader/loader.rb | 9 ++++ cicd/sync/Vagrantfile | 48 ++++++++++++++++++ cicd/sync/client.sh | 3 ++ cicd/sync/config.sh | 3 ++ cicd/sync/ep.sh | 4 ++ cicd/sync/loxilb.sh | 17 +++++++ cicd/sync/rmconfig.sh | 5 ++ cicd/sync/validation.sh | 49 +++++++++++++++++++ 45 files changed, 279 insertions(+) create mode 100644 cicd/sync/.vagrant/bundler/global.sol create mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/action_provision create mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/action_set_name create mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/box_meta create mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/creator_uid create mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/id create mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/index_uuid create mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/private_key create mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/synced_folders create mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/vagrant_cwd create mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/action_provision create mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/action_set_name create mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/box_meta create mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/creator_uid create mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/id create mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/index_uuid create mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/private_key create mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/synced_folders create mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/vagrant_cwd create mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/action_provision create mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/action_set_name create mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/box_meta create mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/creator_uid create mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/id create mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/index_uuid create mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/private_key create mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/synced_folders create mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/vagrant_cwd create mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/action_provision create mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/action_set_name create mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/box_meta create mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/creator_uid create mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/id create mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/index_uuid create mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/private_key create mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/synced_folders create mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/vagrant_cwd create mode 100644 cicd/sync/.vagrant/rgloader/loader.rb create mode 100644 cicd/sync/Vagrantfile create mode 100644 cicd/sync/client.sh create mode 100755 cicd/sync/config.sh create mode 100644 cicd/sync/ep.sh create mode 100644 cicd/sync/loxilb.sh create mode 100755 cicd/sync/rmconfig.sh create mode 100755 cicd/sync/validation.sh diff --git a/cicd/sync/.vagrant/bundler/global.sol b/cicd/sync/.vagrant/bundler/global.sol new file mode 100644 index 000000000..92f433b3d --- /dev/null +++ b/cicd/sync/.vagrant/bundler/global.sol @@ -0,0 +1 @@ +{"dependencies":[["vagrant-disksize",["= 0.1.3"]],["net-ssh",[">= 2.6.5","< 8.0.0"]],["net-scp",[">= 1.1"]],["log4r",["~> 1.1"]],["vagrant-scp",["= 0.5.9"]]],"checksum":"d41cdc087dc2595e62da764647bfcacc91965875ce15159c44cdee684a184f69","vagrant_version":"2.3.6"} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/action_provision b/cicd/sync/.vagrant/machines/client/virtualbox/action_provision new file mode 100644 index 000000000..38b3e6fa0 --- /dev/null +++ b/cicd/sync/.vagrant/machines/client/virtualbox/action_provision @@ -0,0 +1 @@ +1.5:8c652868-9855-401c-86ca-033fe1970bbf \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/action_set_name b/cicd/sync/.vagrant/machines/client/virtualbox/action_set_name new file mode 100644 index 000000000..28652504a --- /dev/null +++ b/cicd/sync/.vagrant/machines/client/virtualbox/action_set_name @@ -0,0 +1 @@ +1691737191 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/box_meta b/cicd/sync/.vagrant/machines/client/virtualbox/box_meta new file mode 100644 index 000000000..4e311f257 --- /dev/null +++ b/cicd/sync/.vagrant/machines/client/virtualbox/box_meta @@ -0,0 +1 @@ +{"name":"sysnet4admin/Ubuntu-k8s","version":"0.7.1","provider":"virtualbox","directory":"boxes/sysnet4admin-VAGRANTSLASH-Ubuntu-k8s/0.7.1/virtualbox"} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/creator_uid b/cicd/sync/.vagrant/machines/client/virtualbox/creator_uid new file mode 100644 index 000000000..e37d32abb --- /dev/null +++ b/cicd/sync/.vagrant/machines/client/virtualbox/creator_uid @@ -0,0 +1 @@ +1000 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/id b/cicd/sync/.vagrant/machines/client/virtualbox/id new file mode 100644 index 000000000..76eba5505 --- /dev/null +++ b/cicd/sync/.vagrant/machines/client/virtualbox/id @@ -0,0 +1 @@ +8c652868-9855-401c-86ca-033fe1970bbf \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/index_uuid b/cicd/sync/.vagrant/machines/client/virtualbox/index_uuid new file mode 100644 index 000000000..4c4e3a465 --- /dev/null +++ b/cicd/sync/.vagrant/machines/client/virtualbox/index_uuid @@ -0,0 +1 @@ +c5c7c415a7ab484980646433ea597d90 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/private_key b/cicd/sync/.vagrant/machines/client/virtualbox/private_key new file mode 100644 index 000000000..575532c1d --- /dev/null +++ b/cicd/sync/.vagrant/machines/client/virtualbox/private_key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEA70YmdMs0gpWIK9Gip2ejKx01Vx8QjL5+gQ2hqrUf4pVR19EY +PY29/vpKHy9XUSkGM7IxLuQEXivEG0m5zjoMv5fFjnGCgeRkHUSKVqujMEikIHmA ++LuN7RYIS/S7IISOhrVz11h1XR7bOT96sISf0AlG0GQOYj9aLJ2pe+BBip2uykKK +vlGyNo1+Zf+WQRGaK5BsOS7YMli+KQOOhmDHLfF2WKc64BIa8JxEGXfaJiik8twi +RuVskdpNvQFGgo8cnjJ751HOUeo6b0eFsDQEy2oIKmght20zWwmpKwf5GuK/XG+9 ++MrdFWkib7UURUtfOTXPEHALixYgcZ/GaZmJJwIDAQABAoIBAQDrPVyxltaZS8oE +yqXPT+TrjlvSu4fZ168n0e8PleMzMPq146yhV6lj9U7KOhDuiha+3IHeR+7DZuP2 +sBUs24WNF/nygUZ9zb1fyPL5IuXdoo06IDrcUAjI/tBJXOJcpfY6Gks5fI5mbt+4 ++N/2i21TastkNfquPPq6wobOczv8ugNL03xkRxTXagVKHmo+AK4llnImMWAIIMjJ +zORsyhUoZd5aTuOV2+lXt+VyRtRiLk0IxUdRTazv20Gn2anmE8YCuu1d2sPd6GwX +lz8hZcPQbm100IEBPAw8zTsJryll7nMhLqmCsy07eP4Ch0R36nvpB8JEnTZT9BwG +xtH9GxkBAoGBAPy/RYwPZmqYyqr5BfKJYb3MgX/mqpkH7FklWINlKYH6XB6pPz6O +fF9T/+b5skFPyly4dwT9eIOKB5SS731W1hyQ0txGtlr12cCWrpE2zByTAojRT22H +7nWjNJL6f/ABU7Je7ttYIz+LuztTt0abpg4sK+KVrcD65g5wjaQU3CwPAoGBAPJa +fQ68Yd3nYXLIDrpMp1uNQY7zNbZCg3eZOlbeJ4+BioI2oeg2H8/gX1jTDnyDNPbU +SnpUFeOCe39+QYVHMczqV5z8b1NZWHZJO1oW+oth1/Z3jhie4gyCxtDOZqYJQOLY +HdzmmIyiIR5/VO5nqdCfrqeVRSPoUbMMsWhvIhlpAoGBALo/oNpTRt9qdH6FP1K4 +1iPtHcjIjBJ0+j+++3joBSxY96VIhuhsS05cMgldG3RxfefRrzMoLbSQ9IERvhWY +9fLa7eeQ3DQvUyVOZJlxzfuIgaecxnEqaBicCdP9cGXGNI0O0+kjC+Z566EOUdJg +RH/RIQ25yu6YdQtS+a1eCdkVAoGAPbrowbkgIHqQKsFkZvWhZ0NqoHO4ia0VeYAx +CPFbGEWBtzsdMF2g6r0W7DQOK47sDbbzJYeHb8TLmiTZoY4J1E0YdecioBNQ4ggz +MvZJaPlKHgBD1Geb4l3pMERyhvYybdVofdzKnH+ree4q/vKuoXIa9Zqi2suRC88+ +3P/5qokCgYEAlHtkIa3fvutcdQJlNXamhbLKsYDlj2rIWB8iL29GbVR7UlJk6qda +bxVE2ERaAT708lK6c0K36rJN3vv0on2Esq2xLRlHO/dv16wWls6ovWLrlQnllYx1 +nkuZKatOqVHOQmPvm9wUOMecIOovPFh11komBLzZKW8+n/t5DJJdT5I= +-----END RSA PRIVATE KEY----- diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/synced_folders b/cicd/sync/.vagrant/machines/client/virtualbox/synced_folders new file mode 100644 index 000000000..38126049a --- /dev/null +++ b/cicd/sync/.vagrant/machines/client/virtualbox/synced_folders @@ -0,0 +1 @@ +{"virtualbox":{"/vagrant":{"guestpath":"/vagrant","hostpath":"/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync","disabled":false,"__vagrantfile":true}}} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/vagrant_cwd b/cicd/sync/.vagrant/machines/client/virtualbox/vagrant_cwd new file mode 100644 index 000000000..7c7e228bb --- /dev/null +++ b/cicd/sync/.vagrant/machines/client/virtualbox/vagrant_cwd @@ -0,0 +1 @@ +/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/action_provision b/cicd/sync/.vagrant/machines/ep1/virtualbox/action_provision new file mode 100644 index 000000000..7fa3f351b --- /dev/null +++ b/cicd/sync/.vagrant/machines/ep1/virtualbox/action_provision @@ -0,0 +1 @@ +1.5:2cfebe94-0185-4665-98b9-5c125727f430 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/action_set_name b/cicd/sync/.vagrant/machines/ep1/virtualbox/action_set_name new file mode 100644 index 000000000..74a55664b --- /dev/null +++ b/cicd/sync/.vagrant/machines/ep1/virtualbox/action_set_name @@ -0,0 +1 @@ +1691737252 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/box_meta b/cicd/sync/.vagrant/machines/ep1/virtualbox/box_meta new file mode 100644 index 000000000..4e311f257 --- /dev/null +++ b/cicd/sync/.vagrant/machines/ep1/virtualbox/box_meta @@ -0,0 +1 @@ +{"name":"sysnet4admin/Ubuntu-k8s","version":"0.7.1","provider":"virtualbox","directory":"boxes/sysnet4admin-VAGRANTSLASH-Ubuntu-k8s/0.7.1/virtualbox"} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/creator_uid b/cicd/sync/.vagrant/machines/ep1/virtualbox/creator_uid new file mode 100644 index 000000000..e37d32abb --- /dev/null +++ b/cicd/sync/.vagrant/machines/ep1/virtualbox/creator_uid @@ -0,0 +1 @@ +1000 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/id b/cicd/sync/.vagrant/machines/ep1/virtualbox/id new file mode 100644 index 000000000..8d8fe1b9b --- /dev/null +++ b/cicd/sync/.vagrant/machines/ep1/virtualbox/id @@ -0,0 +1 @@ +2cfebe94-0185-4665-98b9-5c125727f430 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/index_uuid b/cicd/sync/.vagrant/machines/ep1/virtualbox/index_uuid new file mode 100644 index 000000000..e2b2f8a82 --- /dev/null +++ b/cicd/sync/.vagrant/machines/ep1/virtualbox/index_uuid @@ -0,0 +1 @@ +68f564ec81f248d086a9e998bc4f6c62 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/private_key b/cicd/sync/.vagrant/machines/ep1/virtualbox/private_key new file mode 100644 index 000000000..5864149ad --- /dev/null +++ b/cicd/sync/.vagrant/machines/ep1/virtualbox/private_key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEA2FQz9jtwHNZdqNwKLKPzJfzVk1Jxio5cVNl5tqWYo1kCu9Fo +ZcTfMd0nhc5jwXCZqlU9tISQyNASt+IyoWochFQSjpesU3vqQ60CkyL97RN0xo2a +Kylo3nSyxnbgS8ktLXNVmbch1YImIvvDmjPepZDu8s8n8RC0zjOZBSDLY3uTurHQ +OZueuoV6pDWjKH739T9VB79xzCeZRUlpJT7nxp9glXd2YyCpWj2FI4t+bVeb2vt7 +FG6oziuPZ6bAYw3/iPISMIELlYPFqt57ceBoH+JWmBRsvp8KeZa66pPEjDs3lVr/ +eOM9RLDgrMPVbNia+VGDPiKJTDLY3JEZZkiQvQIDAQABAoIBAEQUk9nNMOZSbF1l +CmqsGix9WnWnv4XccaBeZNifWt8ldx/GsnU8tQVLVVHd9zT8i8+4TdgO1EXufcWT +EV4NN/ZJQyHPB/OqDrrHqwTFQWan9ds62yb0PEWuMkkVRldzc51XK7xZjvNTp07h +LtjLtCkk6J5PpYFPQaeXQ3QCsx+WQ1tANac8dToZ2l2tUQtldJ/+EfiMfF+vUQ9M +GU+slhFdaYyD5lSICWulNgBB9COLG41Xynsi8SOjvrgcB7Y2cnzt47NdjkngWzlG +PATKObhagF2fYHMtuoZ0ouZJlWvD6HgoNywKGAn80YTejOEAHvFzYNRAv2fDHo1H +RqCqjd0CgYEA89/rOEw7+OC8AtkFM3vwd7wnMuly8GKEV+rGpqE4SaMb3YYMVW4Z +7futxjKcnnnVOhMoM4djcOjCrsy716ZgWaiY1gcx50nEJpj6VLfF3HrhXDVNGxCz +fhirTX/RQJ7AULzpI1Htkaa0kfrOM2yzVao6k6PjNhesSzVsMB0rOnsCgYEA4xWt +OWpy/hs8L/h4+eL3FcvCeklQR56kLUzkiPN8671rBh7nOh+h7wUpXNejRIU8qXFL +0Neiqul7d11U74ktrsVsCBOyS0oW9tcwSJlulOWU0oQ+r37mmX70eLAswRpIpGPi +7ii02nJyFGllutSUPF14pARtB9t2TU20XyoLeCcCgYEAuuN7T85kGyqmCgyjxruR +IUDoTUTHl134C6PYjjN4fj3oCdBDPWi9qDm6BuwcR/oXpISvxyVW2F0XbHtq7C0N +I0hnr4UeAh/umKGdHTUkBP2KHG6gY0BePSDMjfyIsFu4B2dtxmpvg7HcKQ6p3JDN +sbVdZG0wnvBvUcdj3E9OJGcCgYEAzveLuN/HTMff1EzfXck5r7jVWIUHUS9KrSr+ +w78/5M/wAlUaI8R+341RxJeY67OPtVNYaVCPprxEVmdPKT5az4aJzta80pgFVBaO +oCyPN9ADEteLFoJPhCjMgk73i0v8Vy/evXeGVe6nAOmK2Xw0WsxKTxrlkJq6OPyQ +X4e/ilMCgYEAm5/o9TF9QliZyNq5z+wjCUuKIik1EKCteN8ZMutBKWBOipAsgOVO +1rfKePFKSSIhUmqE3gllpgaHJ/K8+O7XTc7yj7/ma051hjbuapcNLrJh4dtVHnmV +PRd2RSV/aIrlXLBmw+TgA12bPO1dYTYZP3npUFxoOyKX4Drm9Rvlt7U= +-----END RSA PRIVATE KEY----- diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/synced_folders b/cicd/sync/.vagrant/machines/ep1/virtualbox/synced_folders new file mode 100644 index 000000000..38126049a --- /dev/null +++ b/cicd/sync/.vagrant/machines/ep1/virtualbox/synced_folders @@ -0,0 +1 @@ +{"virtualbox":{"/vagrant":{"guestpath":"/vagrant","hostpath":"/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync","disabled":false,"__vagrantfile":true}}} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/vagrant_cwd b/cicd/sync/.vagrant/machines/ep1/virtualbox/vagrant_cwd new file mode 100644 index 000000000..7c7e228bb --- /dev/null +++ b/cicd/sync/.vagrant/machines/ep1/virtualbox/vagrant_cwd @@ -0,0 +1 @@ +/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/action_provision b/cicd/sync/.vagrant/machines/llb1/virtualbox/action_provision new file mode 100644 index 000000000..5a58ed65e --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb1/virtualbox/action_provision @@ -0,0 +1 @@ +1.5:1e58200b-c394-4229-bd66-d011d78ee3ae \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/action_set_name b/cicd/sync/.vagrant/machines/llb1/virtualbox/action_set_name new file mode 100644 index 000000000..b95dd93e3 --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb1/virtualbox/action_set_name @@ -0,0 +1 @@ +1691736629 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/box_meta b/cicd/sync/.vagrant/machines/llb1/virtualbox/box_meta new file mode 100644 index 000000000..4e311f257 --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb1/virtualbox/box_meta @@ -0,0 +1 @@ +{"name":"sysnet4admin/Ubuntu-k8s","version":"0.7.1","provider":"virtualbox","directory":"boxes/sysnet4admin-VAGRANTSLASH-Ubuntu-k8s/0.7.1/virtualbox"} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/creator_uid b/cicd/sync/.vagrant/machines/llb1/virtualbox/creator_uid new file mode 100644 index 000000000..e37d32abb --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb1/virtualbox/creator_uid @@ -0,0 +1 @@ +1000 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/id b/cicd/sync/.vagrant/machines/llb1/virtualbox/id new file mode 100644 index 000000000..ba8515c77 --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb1/virtualbox/id @@ -0,0 +1 @@ +1e58200b-c394-4229-bd66-d011d78ee3ae \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/index_uuid b/cicd/sync/.vagrant/machines/llb1/virtualbox/index_uuid new file mode 100644 index 000000000..ace297c5b --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb1/virtualbox/index_uuid @@ -0,0 +1 @@ +5e5544b21e464707aaf837e73a9f6df1 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/private_key b/cicd/sync/.vagrant/machines/llb1/virtualbox/private_key new file mode 100644 index 000000000..6eab4d426 --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb1/virtualbox/private_key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA2Dk72G7ip/rnsTbMq7UjOden4Tc+Kh4EgtPMzhs1Af21PEZn +V/Gv1PjDUXXK8uCJI+8wEU2rpiF6N9F2xo+1ZwFNq6BFF8bjLhBK3vamvZl4JilA +kCGpyLWQlSFhCoVtcGhhbXAVpVQSWctoE5uqXLxZFS5ArxLJCGfZOK501NRd3e3K +/TqfX+Mn121vIU1jDOGtMutubmAgyirrDPJUBpyF7Mmq9exr39Cpfklf3ZbK/NLF +Hc6ACjirIM8Oag3E0TzhsLHyLRPaAtxtz9aSPvBWw+YNZLpSzeoEC3p9HY1HHPLH +KeEk5SDCSkIfBmIvzD2GYSALyQQQrLGCx2GyhQIDAQABAoIBAGoOAJWTJedAwP4C +I/egvCt15Kkp4HrmCE5nNpBtdd2JJKjw22WebSEn9x1wimAA1CNlAY2rLCMrTngs +bHbOLCrez8r7rIgSDIIeiaq9DmH28aC+ISXQ7Sc+LzGGksOHWHuHJyp3jJZwGyQi +CeV8N96On58s9ZspuqM91KThpPpyXNQuBmy2Mia0uz0gmJw3rYsGyIqJmJK5UrG1 +i7ZSJ618f2vIQ0+ww60IhNQa6wxIk0JuqtpL1XYoBDToO0ak6oaINzbzuoI2ny2U +LhfOxyO7ong7pGu4Q2CXpraR7t/eXpnI5XcfCLrK9eKlz4bxeU7nkgHU/OkiWjbn +mEtDOoECgYEA7+s/ucBP/Uzgrk7qgT00/SqiR+tclCvDpb9oLkc9N5+rhqrElk10 +HhfFsoIfv1seL0KaXljmyXLaKe8IWGsVfiMngV5XIUmkQVIXDnEGXAV+qMkSdQuz +eMBxJNV+IPlH/M61h6Bf4orohvBhV7xVyqIL4KgOXFYoMD0h8bkeomUCgYEA5rdl +0mWVUyMAjEP3pHsrH4BF4baO1fUHEwRjPR6qWgmmlUGHUmD4JUdQBV7u3jZjC7Eo +cIJikxB4+1ZsMxgYLZ0vprag+3raT00r4vtqsDIUSQGigiAknr8bZaPVuTObcraa +NX6XIyKgRFp6Ti8ZORjz+bslz+yuR9A0VJhFvaECgYEAk4QqIqREipKCc6rCNcTU +l/Vm4qGsws5F9dvCyvGyikj6xkqppWDqTLaBMEkiDB6SaKyjDe/Lt4GO8smub+8n +v+8Nk1XWbPlCrYcz9gU4y8v1HQLE/oK7TzvEVcmPS+b6hE70/vd0/W47mvyiXbfH +aFhKxU+PxOaUv1RXUrFoTKECgYEAuhscPoCE2KA2c+7rlzlniqyNMkpxo7NYdoWm +i4nt/l0ki+dlAhWIdYG10e9HHeafNLI+Hat/WoQCIzQOaZS5BCp0YrWU2+a4rRli +GmGWKPTsmguAjBjjkU41CmMDJrazYPBdKmOoBMaMozacy+6AGSfAviSxlqyj1Pfl +DowaTaECgYAkOTScumu3DfARMLDVDxebO2L07P/KKIlg+YKiORqNBtf0f1xLAPiv +Yle9onNkjlHtWNkIxOAiz21fz9Uxc+ghB2RFU7ZX3WOuAN/zsbPv8R4yYXipuUsP +6cSU31Ywu2Ubp4ZflqaTE5XXNEY0hmLjkm+TEGQbbHaqVGi77ojoGw== +-----END RSA PRIVATE KEY----- diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/synced_folders b/cicd/sync/.vagrant/machines/llb1/virtualbox/synced_folders new file mode 100644 index 000000000..38126049a --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb1/virtualbox/synced_folders @@ -0,0 +1 @@ +{"virtualbox":{"/vagrant":{"guestpath":"/vagrant","hostpath":"/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync","disabled":false,"__vagrantfile":true}}} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/vagrant_cwd b/cicd/sync/.vagrant/machines/llb1/virtualbox/vagrant_cwd new file mode 100644 index 000000000..7c7e228bb --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb1/virtualbox/vagrant_cwd @@ -0,0 +1 @@ +/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/action_provision b/cicd/sync/.vagrant/machines/llb2/virtualbox/action_provision new file mode 100644 index 000000000..f46d920c6 --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb2/virtualbox/action_provision @@ -0,0 +1 @@ +1.5:381e6633-ff00-49ed-85ef-fd688e60e881 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/action_set_name b/cicd/sync/.vagrant/machines/llb2/virtualbox/action_set_name new file mode 100644 index 000000000..44022ce7e --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb2/virtualbox/action_set_name @@ -0,0 +1 @@ +1691736942 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/box_meta b/cicd/sync/.vagrant/machines/llb2/virtualbox/box_meta new file mode 100644 index 000000000..4e311f257 --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb2/virtualbox/box_meta @@ -0,0 +1 @@ +{"name":"sysnet4admin/Ubuntu-k8s","version":"0.7.1","provider":"virtualbox","directory":"boxes/sysnet4admin-VAGRANTSLASH-Ubuntu-k8s/0.7.1/virtualbox"} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/creator_uid b/cicd/sync/.vagrant/machines/llb2/virtualbox/creator_uid new file mode 100644 index 000000000..e37d32abb --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb2/virtualbox/creator_uid @@ -0,0 +1 @@ +1000 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/id b/cicd/sync/.vagrant/machines/llb2/virtualbox/id new file mode 100644 index 000000000..94b1f98dd --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb2/virtualbox/id @@ -0,0 +1 @@ +381e6633-ff00-49ed-85ef-fd688e60e881 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/index_uuid b/cicd/sync/.vagrant/machines/llb2/virtualbox/index_uuid new file mode 100644 index 000000000..5a6f1a2b3 --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb2/virtualbox/index_uuid @@ -0,0 +1 @@ +c29eda8a35d84899b80ff63c64775c03 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/private_key b/cicd/sync/.vagrant/machines/llb2/virtualbox/private_key new file mode 100644 index 000000000..3de083641 --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb2/virtualbox/private_key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAuL4q7lqoaojZFfr9T+EGB9ovWb4yYgHYKneV/r6XLb5eo+8I +DWHbryKZ8LrQrpUcAHT6sjvPamt1bLTWMV220i3rV0IkyA51MyP9xQdJyiwrrcpq +Q7V32XHk56vZ6Tctdg+yoBnwN/jf4rTmcOE5LgyW+zY1EH752sTGvmsWFZZQVqnv +c4gdwVBCOvLEikOCOzauKL9tn8QYZ9FDHj5AmTd6OvM+fzoS2RwP6VB0gYGGKRom +KyX4f01UiKGkEpERP3j69sm+skp2m1aWrgEAzJShSgmTmS/Fjd8rkFusYh/tgQxE +Uml2r4aEE8klD/wc/vMBz+QG6vFoLpQdTYQY4wIDAQABAoIBACMcwbwbrvbQm7H5 +yjtsYWEMrwM3iwvlndg0cmXAg6tGC2bM/5LLjp1yzh9a4l7MKE5N7XgCtiFb2y2W +ET/AuAaqg5hKEgOT++KEbK8N+5BZZNPbLOdBjk3dv8vodbpFqnZN9X1IcMeynZrR +FxtNMzj1W5zTaLKB76sMy1lVsRTY8g6GWsmtuD8QksNVpa3voWc0SUXwVbraBYT9 ++oK5qx80Jm1bhXOVCjvlm/Zq74mHVfg1s+cv+Rv1CG9hluRERwNFjkIW4o2ykVXP +VhzL9Dy8A9RKGd9gz29buD5+NtEA9wiRucfXnKMpyhlo5q8fR5RcAOkV95J+Bj8B +y+SzIkECgYEA5ktZoJ89abWti3L059dYbfVCZYI/qQP9s5SDo0lwcsqodDrTneuN +4e1USl0k9fTTiiGzF+IxHbvoqpiyPI0+iBtGGzSkxL//k2/n831kp8BD0O+oD/EM +AFJvdSVQW/KD/XUfyPUPTKyQ4MJNB1dOcLjq2t/Bnkvz7icJLsHIuI0CgYEAzV0v +j8YkMJOX0yU1ajCW6Q4Zbr/zcYLz7iKlRQ7CApE+1fWPiefNxwQeGCrbTM2l8DM6 +Fb9sN2om6qCkyzPet65hiZw48KUTK1d7i8HqGL8In1mZ8pvq5+vK+SBuBVALk8aw +F4e+XhNE8a4nxHEIn4nc91EACBkJyvwIUt9G0y8CgYA4BeaU24guGiNQxDNt3wZF +IZ/4X7oq3KywLhXj+C6hzkkylUj7pt0KNBkISiZ+qwN3HAGccssuB5NvRNw/PMEU +jUX9wk2/C1IaROPxaPS+llgpENQapLIdtfAlWGF3LMUNOJv14B/LsENCWiIxAf00 +ENgZTMumPaXmrppeRIgW1QKBgCbVjU3p0SQidN5ntd8byx3NHcAUqkt30PriMFCL +xuJHZxN8uSUB4abVv38gc5HlfZCbj0tXhHZCK5PIxwDqER646hbkpNvUy3gxtdoq +1ELTtSS1A+id1WD/9MUBfeyhQ18xflEkyTnu+OWe4kRCf7fzkQZvmcruOSzaVhSd +xj3fAoGAfyHK6RgZuH59eyEKaSoCeXn+V+MVx407rxauCIMncHTEnysouWxalwX2 +mZW75xmOzwtvoXdQBlFpD1RozMAKHx7nDNavY04IYh7p4t66OKG7axP/P/8fpwGW +6z+DjGHJeQkEanEncs57mWHSNhqhBJI2amGyT0sYLwDaBzaX1SA= +-----END RSA PRIVATE KEY----- diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/synced_folders b/cicd/sync/.vagrant/machines/llb2/virtualbox/synced_folders new file mode 100644 index 000000000..38126049a --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb2/virtualbox/synced_folders @@ -0,0 +1 @@ +{"virtualbox":{"/vagrant":{"guestpath":"/vagrant","hostpath":"/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync","disabled":false,"__vagrantfile":true}}} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/vagrant_cwd b/cicd/sync/.vagrant/machines/llb2/virtualbox/vagrant_cwd new file mode 100644 index 000000000..7c7e228bb --- /dev/null +++ b/cicd/sync/.vagrant/machines/llb2/virtualbox/vagrant_cwd @@ -0,0 +1 @@ +/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync \ No newline at end of file diff --git a/cicd/sync/.vagrant/rgloader/loader.rb b/cicd/sync/.vagrant/rgloader/loader.rb new file mode 100644 index 000000000..c3c05b095 --- /dev/null +++ b/cicd/sync/.vagrant/rgloader/loader.rb @@ -0,0 +1,9 @@ +# This file loads the proper rgloader/loader.rb file that comes packaged +# with Vagrant so that encoded files can properly run with Vagrant. + +if ENV["VAGRANT_INSTALLER_EMBEDDED_DIR"] + require File.expand_path( + "rgloader/loader", ENV["VAGRANT_INSTALLER_EMBEDDED_DIR"]) +else + raise "Encoded files can't be read outside of the Vagrant installer." +end diff --git a/cicd/sync/Vagrantfile b/cicd/sync/Vagrantfile new file mode 100644 index 000000000..d097a9fc2 --- /dev/null +++ b/cicd/sync/Vagrantfile @@ -0,0 +1,48 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +loxilbs = (ENV['LOXILBS'] || "2").to_i +eps = (ENV['LOXILBS'] || "1").to_i +box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s") +box_version = "0.7.1" +Vagrant.configure("2") do |config| + config.vm.box = "#{box_name}" + config.vm.box_version = "#{box_version}" + + (1..loxilbs).each do |node_number| + config.vm.define "llb#{node_number}" do |loxilb| + loxilb.vm.hostname = "llb#{node_number}" + ip = node_number + 10 + loxilb.vm.network :private_network, ip: "192.168.80.#{ip}", :netmask => "255.255.255.0" + loxilb.vm.network :private_network, ip: "192.168.90.#{ip}", :netmask => "255.255.255.0" + loxilb.vm.provision :shell, :path => "loxilb.sh" + loxilb.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 6000] + vbox.customize ["modifyvm", :id, "--cpus", 4] + end + end + end + + config.vm.define "client" do |client| + client.vm.hostname = 'client' + client.vm.network :private_network, ip: "192.168.80.100", :netmask => "255.255.255.0" + client.vm.provision :shell, :path => "client.sh" + client.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 6000] + vbox.customize ["modifyvm", :id, "--cpus", 4] + end + end + + (1..eps).each do |node_number| + config.vm.define "ep#{node_number}" do |ep| + ep.vm.hostname = "ep#{node_number}" + ip = node_number + 100 + ep.vm.network :private_network, ip: "192.168.90.#{ip}", :netmask => "255.255.255.0" + ep.vm.provision :shell, :path => "ep.sh" + ep.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 6000] + vbox.customize ["modifyvm", :id, "--cpus", 4] + end + end + end +end diff --git a/cicd/sync/client.sh b/cicd/sync/client.sh new file mode 100644 index 000000000..8eea07ef0 --- /dev/null +++ b/cicd/sync/client.sh @@ -0,0 +1,3 @@ +sudo apt -y install autoconf automake libtool bison flex gcc ncurses-dev +git clone https://github.com/satori-com/tcpkali.git +sudo ip route add 20.20.20.1 via 192.168.80.11 diff --git a/cicd/sync/config.sh b/cicd/sync/config.sh new file mode 100755 index 000000000..6b8ee48ef --- /dev/null +++ b/cicd/sync/config.sh @@ -0,0 +1,3 @@ +#!/bin/bash +vagrant global-status | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f +vagrant up diff --git a/cicd/sync/ep.sh b/cicd/sync/ep.sh new file mode 100644 index 000000000..e838b8cc4 --- /dev/null +++ b/cicd/sync/ep.sh @@ -0,0 +1,4 @@ +sudo ip route add 20.20.20.1 via 192.168.90.11 +sudo ip route add 192.168.80.0/24 via 192.168.90.11 +sudo apt -y install autoconf automake libtool bison flex gcc ncurses-dev +git clone https://github.com/satori-com/tcpkali.git diff --git a/cicd/sync/loxilb.sh b/cicd/sync/loxilb.sh new file mode 100644 index 000000000..02b7d1501 --- /dev/null +++ b/cicd/sync/loxilb.sh @@ -0,0 +1,17 @@ +export LOXILB_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.80' | awk '{print $2}' | cut -f1 -d '/') + +apt-get update +apt-get install -y software-properties-common +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get install -y docker-ce +docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged --entrypoint /bin/bash -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest + +docker exec -it loxilb apt update +docker exec -it loxilb apt -y install clang-10 llvm libelf-dev gcc-multilib libpcap-dev linux-tools-$(uname -r) elfutils dwarves git libbsd-dev bridge-utils unzip build-essential bison flex iperf iproute2 nodejs socat ethtool +docker exec -it loxilb git clone https://github.com/loxilb-io/loxilb.git --recurse-submodules + +echo alias loxicmd=\"sudo docker exec -it loxilb loxicmd\" >> ~/.bashrc +echo alias loxilb=\"sudo docker exec -it loxilb \" >> ~/.bashrc + +echo $LOXILB_IP > /vagrant/loxilb-$(hostname) diff --git a/cicd/sync/rmconfig.sh b/cicd/sync/rmconfig.sh new file mode 100755 index 000000000..3fe1be339 --- /dev/null +++ b/cicd/sync/rmconfig.sh @@ -0,0 +1,5 @@ +#!/bin/bash +vagrant destroy -f ep1 +vagrant destroy -f client +vagrant destroy -f llb1 +vagrant destroy -f llb2 diff --git a/cicd/sync/validation.sh b/cicd/sync/validation.sh new file mode 100755 index 000000000..a8fc56fef --- /dev/null +++ b/cicd/sync/validation.sh @@ -0,0 +1,49 @@ +#!/bin/bash +source ../common.sh +echo k3s-cilium-cluster + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +# Set space as the delimiter +IFS=' ' + +sleep 45 +extIP="123.123.123.1" +echo $extIP + +echo "Service Info" +vagrant ssh master -c 'sudo kubectl get svc' +echo "LB Info" +vagrant ssh loxilb -c 'sudo docker exec -i loxilb loxicmd get lb -o wide' +echo "EP Info" +vagrant ssh loxilb -c 'sudo docker exec -i loxilb loxicmd get ep -o wide' + +print_debug_info() { + echo "llb1 route-info" + vagrant ssh loxilb -c 'ip route' + vagrant ssh master -c 'sudo kubectl get pods -A' + vagrant ssh master -c 'sudo kubectl get svc' + vagrant ssh master -c 'sudo kubectl get nodes' +} + +out=$(curl -s --connect-timeout 10 http://$extIP:55002) +if [[ ${out} == *"Welcome to nginx"* ]]; then + echo "k3s-cilium-cluster (kube-loxilb) tcp [OK]" +else + echo "k3s-cilium-cluster (kube-loxilb) tcp [FAILED]" + print_debug_info + exit 1 +fi + +out=$(timeout 10 ../common/udp_client $extIP 55003) +if [[ ${out} == *"Client"* ]]; then + echo "k3s-cilium-cluster (kube-loxilb) udp [OK]" +else + echo "k3s-cilium-cluster (kube-loxilb) udp [FAILED]" + print_debug_info + exit 1 +fi + +exit From 8824d3dd0bd43749d774ab3a4e9854f28d8836e7 Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Wed, 16 Aug 2023 17:17:16 +0900 Subject: [PATCH 2/8] removed unwanted files --- cicd/sync/.vagrant/bundler/global.sol | 1 - .../client/virtualbox/action_provision | 1 - .../client/virtualbox/action_set_name | 1 - .../machines/client/virtualbox/box_meta | 1 - .../machines/client/virtualbox/creator_uid | 1 - .../.vagrant/machines/client/virtualbox/id | 1 - .../machines/client/virtualbox/index_uuid | 1 - .../machines/client/virtualbox/private_key | 27 ------------------- .../machines/client/virtualbox/synced_folders | 1 - .../machines/client/virtualbox/vagrant_cwd | 1 - .../machines/ep1/virtualbox/action_provision | 1 - .../machines/ep1/virtualbox/action_set_name | 1 - .../.vagrant/machines/ep1/virtualbox/box_meta | 1 - .../machines/ep1/virtualbox/creator_uid | 1 - cicd/sync/.vagrant/machines/ep1/virtualbox/id | 1 - .../machines/ep1/virtualbox/index_uuid | 1 - .../machines/ep1/virtualbox/private_key | 27 ------------------- .../machines/ep1/virtualbox/synced_folders | 1 - .../machines/ep1/virtualbox/vagrant_cwd | 1 - .../machines/llb1/virtualbox/action_provision | 1 - .../machines/llb1/virtualbox/action_set_name | 1 - .../machines/llb1/virtualbox/box_meta | 1 - .../machines/llb1/virtualbox/creator_uid | 1 - .../sync/.vagrant/machines/llb1/virtualbox/id | 1 - .../machines/llb1/virtualbox/index_uuid | 1 - .../machines/llb1/virtualbox/private_key | 27 ------------------- .../machines/llb1/virtualbox/synced_folders | 1 - .../machines/llb1/virtualbox/vagrant_cwd | 1 - .../machines/llb2/virtualbox/action_provision | 1 - .../machines/llb2/virtualbox/action_set_name | 1 - .../machines/llb2/virtualbox/box_meta | 1 - .../machines/llb2/virtualbox/creator_uid | 1 - .../sync/.vagrant/machines/llb2/virtualbox/id | 1 - .../machines/llb2/virtualbox/index_uuid | 1 - .../machines/llb2/virtualbox/private_key | 27 ------------------- .../machines/llb2/virtualbox/synced_folders | 1 - .../machines/llb2/virtualbox/vagrant_cwd | 1 - cicd/sync/.vagrant/rgloader/loader.rb | 9 ------- 38 files changed, 150 deletions(-) delete mode 100644 cicd/sync/.vagrant/bundler/global.sol delete mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/action_provision delete mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/action_set_name delete mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/box_meta delete mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/creator_uid delete mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/id delete mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/index_uuid delete mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/private_key delete mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/synced_folders delete mode 100644 cicd/sync/.vagrant/machines/client/virtualbox/vagrant_cwd delete mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/action_provision delete mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/action_set_name delete mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/box_meta delete mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/creator_uid delete mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/id delete mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/index_uuid delete mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/private_key delete mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/synced_folders delete mode 100644 cicd/sync/.vagrant/machines/ep1/virtualbox/vagrant_cwd delete mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/action_provision delete mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/action_set_name delete mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/box_meta delete mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/creator_uid delete mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/id delete mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/index_uuid delete mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/private_key delete mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/synced_folders delete mode 100644 cicd/sync/.vagrant/machines/llb1/virtualbox/vagrant_cwd delete mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/action_provision delete mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/action_set_name delete mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/box_meta delete mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/creator_uid delete mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/id delete mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/index_uuid delete mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/private_key delete mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/synced_folders delete mode 100644 cicd/sync/.vagrant/machines/llb2/virtualbox/vagrant_cwd delete mode 100644 cicd/sync/.vagrant/rgloader/loader.rb diff --git a/cicd/sync/.vagrant/bundler/global.sol b/cicd/sync/.vagrant/bundler/global.sol deleted file mode 100644 index 92f433b3d..000000000 --- a/cicd/sync/.vagrant/bundler/global.sol +++ /dev/null @@ -1 +0,0 @@ -{"dependencies":[["vagrant-disksize",["= 0.1.3"]],["net-ssh",[">= 2.6.5","< 8.0.0"]],["net-scp",[">= 1.1"]],["log4r",["~> 1.1"]],["vagrant-scp",["= 0.5.9"]]],"checksum":"d41cdc087dc2595e62da764647bfcacc91965875ce15159c44cdee684a184f69","vagrant_version":"2.3.6"} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/action_provision b/cicd/sync/.vagrant/machines/client/virtualbox/action_provision deleted file mode 100644 index 38b3e6fa0..000000000 --- a/cicd/sync/.vagrant/machines/client/virtualbox/action_provision +++ /dev/null @@ -1 +0,0 @@ -1.5:8c652868-9855-401c-86ca-033fe1970bbf \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/action_set_name b/cicd/sync/.vagrant/machines/client/virtualbox/action_set_name deleted file mode 100644 index 28652504a..000000000 --- a/cicd/sync/.vagrant/machines/client/virtualbox/action_set_name +++ /dev/null @@ -1 +0,0 @@ -1691737191 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/box_meta b/cicd/sync/.vagrant/machines/client/virtualbox/box_meta deleted file mode 100644 index 4e311f257..000000000 --- a/cicd/sync/.vagrant/machines/client/virtualbox/box_meta +++ /dev/null @@ -1 +0,0 @@ -{"name":"sysnet4admin/Ubuntu-k8s","version":"0.7.1","provider":"virtualbox","directory":"boxes/sysnet4admin-VAGRANTSLASH-Ubuntu-k8s/0.7.1/virtualbox"} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/creator_uid b/cicd/sync/.vagrant/machines/client/virtualbox/creator_uid deleted file mode 100644 index e37d32abb..000000000 --- a/cicd/sync/.vagrant/machines/client/virtualbox/creator_uid +++ /dev/null @@ -1 +0,0 @@ -1000 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/id b/cicd/sync/.vagrant/machines/client/virtualbox/id deleted file mode 100644 index 76eba5505..000000000 --- a/cicd/sync/.vagrant/machines/client/virtualbox/id +++ /dev/null @@ -1 +0,0 @@ -8c652868-9855-401c-86ca-033fe1970bbf \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/index_uuid b/cicd/sync/.vagrant/machines/client/virtualbox/index_uuid deleted file mode 100644 index 4c4e3a465..000000000 --- a/cicd/sync/.vagrant/machines/client/virtualbox/index_uuid +++ /dev/null @@ -1 +0,0 @@ -c5c7c415a7ab484980646433ea597d90 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/private_key b/cicd/sync/.vagrant/machines/client/virtualbox/private_key deleted file mode 100644 index 575532c1d..000000000 --- a/cicd/sync/.vagrant/machines/client/virtualbox/private_key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpQIBAAKCAQEA70YmdMs0gpWIK9Gip2ejKx01Vx8QjL5+gQ2hqrUf4pVR19EY -PY29/vpKHy9XUSkGM7IxLuQEXivEG0m5zjoMv5fFjnGCgeRkHUSKVqujMEikIHmA -+LuN7RYIS/S7IISOhrVz11h1XR7bOT96sISf0AlG0GQOYj9aLJ2pe+BBip2uykKK -vlGyNo1+Zf+WQRGaK5BsOS7YMli+KQOOhmDHLfF2WKc64BIa8JxEGXfaJiik8twi -RuVskdpNvQFGgo8cnjJ751HOUeo6b0eFsDQEy2oIKmght20zWwmpKwf5GuK/XG+9 -+MrdFWkib7UURUtfOTXPEHALixYgcZ/GaZmJJwIDAQABAoIBAQDrPVyxltaZS8oE -yqXPT+TrjlvSu4fZ168n0e8PleMzMPq146yhV6lj9U7KOhDuiha+3IHeR+7DZuP2 -sBUs24WNF/nygUZ9zb1fyPL5IuXdoo06IDrcUAjI/tBJXOJcpfY6Gks5fI5mbt+4 -+N/2i21TastkNfquPPq6wobOczv8ugNL03xkRxTXagVKHmo+AK4llnImMWAIIMjJ -zORsyhUoZd5aTuOV2+lXt+VyRtRiLk0IxUdRTazv20Gn2anmE8YCuu1d2sPd6GwX -lz8hZcPQbm100IEBPAw8zTsJryll7nMhLqmCsy07eP4Ch0R36nvpB8JEnTZT9BwG -xtH9GxkBAoGBAPy/RYwPZmqYyqr5BfKJYb3MgX/mqpkH7FklWINlKYH6XB6pPz6O -fF9T/+b5skFPyly4dwT9eIOKB5SS731W1hyQ0txGtlr12cCWrpE2zByTAojRT22H -7nWjNJL6f/ABU7Je7ttYIz+LuztTt0abpg4sK+KVrcD65g5wjaQU3CwPAoGBAPJa -fQ68Yd3nYXLIDrpMp1uNQY7zNbZCg3eZOlbeJ4+BioI2oeg2H8/gX1jTDnyDNPbU -SnpUFeOCe39+QYVHMczqV5z8b1NZWHZJO1oW+oth1/Z3jhie4gyCxtDOZqYJQOLY -HdzmmIyiIR5/VO5nqdCfrqeVRSPoUbMMsWhvIhlpAoGBALo/oNpTRt9qdH6FP1K4 -1iPtHcjIjBJ0+j+++3joBSxY96VIhuhsS05cMgldG3RxfefRrzMoLbSQ9IERvhWY -9fLa7eeQ3DQvUyVOZJlxzfuIgaecxnEqaBicCdP9cGXGNI0O0+kjC+Z566EOUdJg -RH/RIQ25yu6YdQtS+a1eCdkVAoGAPbrowbkgIHqQKsFkZvWhZ0NqoHO4ia0VeYAx -CPFbGEWBtzsdMF2g6r0W7DQOK47sDbbzJYeHb8TLmiTZoY4J1E0YdecioBNQ4ggz -MvZJaPlKHgBD1Geb4l3pMERyhvYybdVofdzKnH+ree4q/vKuoXIa9Zqi2suRC88+ -3P/5qokCgYEAlHtkIa3fvutcdQJlNXamhbLKsYDlj2rIWB8iL29GbVR7UlJk6qda -bxVE2ERaAT708lK6c0K36rJN3vv0on2Esq2xLRlHO/dv16wWls6ovWLrlQnllYx1 -nkuZKatOqVHOQmPvm9wUOMecIOovPFh11komBLzZKW8+n/t5DJJdT5I= ------END RSA PRIVATE KEY----- diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/synced_folders b/cicd/sync/.vagrant/machines/client/virtualbox/synced_folders deleted file mode 100644 index 38126049a..000000000 --- a/cicd/sync/.vagrant/machines/client/virtualbox/synced_folders +++ /dev/null @@ -1 +0,0 @@ -{"virtualbox":{"/vagrant":{"guestpath":"/vagrant","hostpath":"/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync","disabled":false,"__vagrantfile":true}}} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/client/virtualbox/vagrant_cwd b/cicd/sync/.vagrant/machines/client/virtualbox/vagrant_cwd deleted file mode 100644 index 7c7e228bb..000000000 --- a/cicd/sync/.vagrant/machines/client/virtualbox/vagrant_cwd +++ /dev/null @@ -1 +0,0 @@ -/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/action_provision b/cicd/sync/.vagrant/machines/ep1/virtualbox/action_provision deleted file mode 100644 index 7fa3f351b..000000000 --- a/cicd/sync/.vagrant/machines/ep1/virtualbox/action_provision +++ /dev/null @@ -1 +0,0 @@ -1.5:2cfebe94-0185-4665-98b9-5c125727f430 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/action_set_name b/cicd/sync/.vagrant/machines/ep1/virtualbox/action_set_name deleted file mode 100644 index 74a55664b..000000000 --- a/cicd/sync/.vagrant/machines/ep1/virtualbox/action_set_name +++ /dev/null @@ -1 +0,0 @@ -1691737252 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/box_meta b/cicd/sync/.vagrant/machines/ep1/virtualbox/box_meta deleted file mode 100644 index 4e311f257..000000000 --- a/cicd/sync/.vagrant/machines/ep1/virtualbox/box_meta +++ /dev/null @@ -1 +0,0 @@ -{"name":"sysnet4admin/Ubuntu-k8s","version":"0.7.1","provider":"virtualbox","directory":"boxes/sysnet4admin-VAGRANTSLASH-Ubuntu-k8s/0.7.1/virtualbox"} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/creator_uid b/cicd/sync/.vagrant/machines/ep1/virtualbox/creator_uid deleted file mode 100644 index e37d32abb..000000000 --- a/cicd/sync/.vagrant/machines/ep1/virtualbox/creator_uid +++ /dev/null @@ -1 +0,0 @@ -1000 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/id b/cicd/sync/.vagrant/machines/ep1/virtualbox/id deleted file mode 100644 index 8d8fe1b9b..000000000 --- a/cicd/sync/.vagrant/machines/ep1/virtualbox/id +++ /dev/null @@ -1 +0,0 @@ -2cfebe94-0185-4665-98b9-5c125727f430 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/index_uuid b/cicd/sync/.vagrant/machines/ep1/virtualbox/index_uuid deleted file mode 100644 index e2b2f8a82..000000000 --- a/cicd/sync/.vagrant/machines/ep1/virtualbox/index_uuid +++ /dev/null @@ -1 +0,0 @@ -68f564ec81f248d086a9e998bc4f6c62 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/private_key b/cicd/sync/.vagrant/machines/ep1/virtualbox/private_key deleted file mode 100644 index 5864149ad..000000000 --- a/cicd/sync/.vagrant/machines/ep1/virtualbox/private_key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpQIBAAKCAQEA2FQz9jtwHNZdqNwKLKPzJfzVk1Jxio5cVNl5tqWYo1kCu9Fo -ZcTfMd0nhc5jwXCZqlU9tISQyNASt+IyoWochFQSjpesU3vqQ60CkyL97RN0xo2a -Kylo3nSyxnbgS8ktLXNVmbch1YImIvvDmjPepZDu8s8n8RC0zjOZBSDLY3uTurHQ -OZueuoV6pDWjKH739T9VB79xzCeZRUlpJT7nxp9glXd2YyCpWj2FI4t+bVeb2vt7 -FG6oziuPZ6bAYw3/iPISMIELlYPFqt57ceBoH+JWmBRsvp8KeZa66pPEjDs3lVr/ -eOM9RLDgrMPVbNia+VGDPiKJTDLY3JEZZkiQvQIDAQABAoIBAEQUk9nNMOZSbF1l -CmqsGix9WnWnv4XccaBeZNifWt8ldx/GsnU8tQVLVVHd9zT8i8+4TdgO1EXufcWT -EV4NN/ZJQyHPB/OqDrrHqwTFQWan9ds62yb0PEWuMkkVRldzc51XK7xZjvNTp07h -LtjLtCkk6J5PpYFPQaeXQ3QCsx+WQ1tANac8dToZ2l2tUQtldJ/+EfiMfF+vUQ9M -GU+slhFdaYyD5lSICWulNgBB9COLG41Xynsi8SOjvrgcB7Y2cnzt47NdjkngWzlG -PATKObhagF2fYHMtuoZ0ouZJlWvD6HgoNywKGAn80YTejOEAHvFzYNRAv2fDHo1H -RqCqjd0CgYEA89/rOEw7+OC8AtkFM3vwd7wnMuly8GKEV+rGpqE4SaMb3YYMVW4Z -7futxjKcnnnVOhMoM4djcOjCrsy716ZgWaiY1gcx50nEJpj6VLfF3HrhXDVNGxCz -fhirTX/RQJ7AULzpI1Htkaa0kfrOM2yzVao6k6PjNhesSzVsMB0rOnsCgYEA4xWt -OWpy/hs8L/h4+eL3FcvCeklQR56kLUzkiPN8671rBh7nOh+h7wUpXNejRIU8qXFL -0Neiqul7d11U74ktrsVsCBOyS0oW9tcwSJlulOWU0oQ+r37mmX70eLAswRpIpGPi -7ii02nJyFGllutSUPF14pARtB9t2TU20XyoLeCcCgYEAuuN7T85kGyqmCgyjxruR -IUDoTUTHl134C6PYjjN4fj3oCdBDPWi9qDm6BuwcR/oXpISvxyVW2F0XbHtq7C0N -I0hnr4UeAh/umKGdHTUkBP2KHG6gY0BePSDMjfyIsFu4B2dtxmpvg7HcKQ6p3JDN -sbVdZG0wnvBvUcdj3E9OJGcCgYEAzveLuN/HTMff1EzfXck5r7jVWIUHUS9KrSr+ -w78/5M/wAlUaI8R+341RxJeY67OPtVNYaVCPprxEVmdPKT5az4aJzta80pgFVBaO -oCyPN9ADEteLFoJPhCjMgk73i0v8Vy/evXeGVe6nAOmK2Xw0WsxKTxrlkJq6OPyQ -X4e/ilMCgYEAm5/o9TF9QliZyNq5z+wjCUuKIik1EKCteN8ZMutBKWBOipAsgOVO -1rfKePFKSSIhUmqE3gllpgaHJ/K8+O7XTc7yj7/ma051hjbuapcNLrJh4dtVHnmV -PRd2RSV/aIrlXLBmw+TgA12bPO1dYTYZP3npUFxoOyKX4Drm9Rvlt7U= ------END RSA PRIVATE KEY----- diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/synced_folders b/cicd/sync/.vagrant/machines/ep1/virtualbox/synced_folders deleted file mode 100644 index 38126049a..000000000 --- a/cicd/sync/.vagrant/machines/ep1/virtualbox/synced_folders +++ /dev/null @@ -1 +0,0 @@ -{"virtualbox":{"/vagrant":{"guestpath":"/vagrant","hostpath":"/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync","disabled":false,"__vagrantfile":true}}} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/ep1/virtualbox/vagrant_cwd b/cicd/sync/.vagrant/machines/ep1/virtualbox/vagrant_cwd deleted file mode 100644 index 7c7e228bb..000000000 --- a/cicd/sync/.vagrant/machines/ep1/virtualbox/vagrant_cwd +++ /dev/null @@ -1 +0,0 @@ -/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/action_provision b/cicd/sync/.vagrant/machines/llb1/virtualbox/action_provision deleted file mode 100644 index 5a58ed65e..000000000 --- a/cicd/sync/.vagrant/machines/llb1/virtualbox/action_provision +++ /dev/null @@ -1 +0,0 @@ -1.5:1e58200b-c394-4229-bd66-d011d78ee3ae \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/action_set_name b/cicd/sync/.vagrant/machines/llb1/virtualbox/action_set_name deleted file mode 100644 index b95dd93e3..000000000 --- a/cicd/sync/.vagrant/machines/llb1/virtualbox/action_set_name +++ /dev/null @@ -1 +0,0 @@ -1691736629 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/box_meta b/cicd/sync/.vagrant/machines/llb1/virtualbox/box_meta deleted file mode 100644 index 4e311f257..000000000 --- a/cicd/sync/.vagrant/machines/llb1/virtualbox/box_meta +++ /dev/null @@ -1 +0,0 @@ -{"name":"sysnet4admin/Ubuntu-k8s","version":"0.7.1","provider":"virtualbox","directory":"boxes/sysnet4admin-VAGRANTSLASH-Ubuntu-k8s/0.7.1/virtualbox"} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/creator_uid b/cicd/sync/.vagrant/machines/llb1/virtualbox/creator_uid deleted file mode 100644 index e37d32abb..000000000 --- a/cicd/sync/.vagrant/machines/llb1/virtualbox/creator_uid +++ /dev/null @@ -1 +0,0 @@ -1000 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/id b/cicd/sync/.vagrant/machines/llb1/virtualbox/id deleted file mode 100644 index ba8515c77..000000000 --- a/cicd/sync/.vagrant/machines/llb1/virtualbox/id +++ /dev/null @@ -1 +0,0 @@ -1e58200b-c394-4229-bd66-d011d78ee3ae \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/index_uuid b/cicd/sync/.vagrant/machines/llb1/virtualbox/index_uuid deleted file mode 100644 index ace297c5b..000000000 --- a/cicd/sync/.vagrant/machines/llb1/virtualbox/index_uuid +++ /dev/null @@ -1 +0,0 @@ -5e5544b21e464707aaf837e73a9f6df1 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/private_key b/cicd/sync/.vagrant/machines/llb1/virtualbox/private_key deleted file mode 100644 index 6eab4d426..000000000 --- a/cicd/sync/.vagrant/machines/llb1/virtualbox/private_key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEA2Dk72G7ip/rnsTbMq7UjOden4Tc+Kh4EgtPMzhs1Af21PEZn -V/Gv1PjDUXXK8uCJI+8wEU2rpiF6N9F2xo+1ZwFNq6BFF8bjLhBK3vamvZl4JilA -kCGpyLWQlSFhCoVtcGhhbXAVpVQSWctoE5uqXLxZFS5ArxLJCGfZOK501NRd3e3K -/TqfX+Mn121vIU1jDOGtMutubmAgyirrDPJUBpyF7Mmq9exr39Cpfklf3ZbK/NLF -Hc6ACjirIM8Oag3E0TzhsLHyLRPaAtxtz9aSPvBWw+YNZLpSzeoEC3p9HY1HHPLH -KeEk5SDCSkIfBmIvzD2GYSALyQQQrLGCx2GyhQIDAQABAoIBAGoOAJWTJedAwP4C -I/egvCt15Kkp4HrmCE5nNpBtdd2JJKjw22WebSEn9x1wimAA1CNlAY2rLCMrTngs -bHbOLCrez8r7rIgSDIIeiaq9DmH28aC+ISXQ7Sc+LzGGksOHWHuHJyp3jJZwGyQi -CeV8N96On58s9ZspuqM91KThpPpyXNQuBmy2Mia0uz0gmJw3rYsGyIqJmJK5UrG1 -i7ZSJ618f2vIQ0+ww60IhNQa6wxIk0JuqtpL1XYoBDToO0ak6oaINzbzuoI2ny2U -LhfOxyO7ong7pGu4Q2CXpraR7t/eXpnI5XcfCLrK9eKlz4bxeU7nkgHU/OkiWjbn -mEtDOoECgYEA7+s/ucBP/Uzgrk7qgT00/SqiR+tclCvDpb9oLkc9N5+rhqrElk10 -HhfFsoIfv1seL0KaXljmyXLaKe8IWGsVfiMngV5XIUmkQVIXDnEGXAV+qMkSdQuz -eMBxJNV+IPlH/M61h6Bf4orohvBhV7xVyqIL4KgOXFYoMD0h8bkeomUCgYEA5rdl -0mWVUyMAjEP3pHsrH4BF4baO1fUHEwRjPR6qWgmmlUGHUmD4JUdQBV7u3jZjC7Eo -cIJikxB4+1ZsMxgYLZ0vprag+3raT00r4vtqsDIUSQGigiAknr8bZaPVuTObcraa -NX6XIyKgRFp6Ti8ZORjz+bslz+yuR9A0VJhFvaECgYEAk4QqIqREipKCc6rCNcTU -l/Vm4qGsws5F9dvCyvGyikj6xkqppWDqTLaBMEkiDB6SaKyjDe/Lt4GO8smub+8n -v+8Nk1XWbPlCrYcz9gU4y8v1HQLE/oK7TzvEVcmPS+b6hE70/vd0/W47mvyiXbfH -aFhKxU+PxOaUv1RXUrFoTKECgYEAuhscPoCE2KA2c+7rlzlniqyNMkpxo7NYdoWm -i4nt/l0ki+dlAhWIdYG10e9HHeafNLI+Hat/WoQCIzQOaZS5BCp0YrWU2+a4rRli -GmGWKPTsmguAjBjjkU41CmMDJrazYPBdKmOoBMaMozacy+6AGSfAviSxlqyj1Pfl -DowaTaECgYAkOTScumu3DfARMLDVDxebO2L07P/KKIlg+YKiORqNBtf0f1xLAPiv -Yle9onNkjlHtWNkIxOAiz21fz9Uxc+ghB2RFU7ZX3WOuAN/zsbPv8R4yYXipuUsP -6cSU31Ywu2Ubp4ZflqaTE5XXNEY0hmLjkm+TEGQbbHaqVGi77ojoGw== ------END RSA PRIVATE KEY----- diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/synced_folders b/cicd/sync/.vagrant/machines/llb1/virtualbox/synced_folders deleted file mode 100644 index 38126049a..000000000 --- a/cicd/sync/.vagrant/machines/llb1/virtualbox/synced_folders +++ /dev/null @@ -1 +0,0 @@ -{"virtualbox":{"/vagrant":{"guestpath":"/vagrant","hostpath":"/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync","disabled":false,"__vagrantfile":true}}} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb1/virtualbox/vagrant_cwd b/cicd/sync/.vagrant/machines/llb1/virtualbox/vagrant_cwd deleted file mode 100644 index 7c7e228bb..000000000 --- a/cicd/sync/.vagrant/machines/llb1/virtualbox/vagrant_cwd +++ /dev/null @@ -1 +0,0 @@ -/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/action_provision b/cicd/sync/.vagrant/machines/llb2/virtualbox/action_provision deleted file mode 100644 index f46d920c6..000000000 --- a/cicd/sync/.vagrant/machines/llb2/virtualbox/action_provision +++ /dev/null @@ -1 +0,0 @@ -1.5:381e6633-ff00-49ed-85ef-fd688e60e881 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/action_set_name b/cicd/sync/.vagrant/machines/llb2/virtualbox/action_set_name deleted file mode 100644 index 44022ce7e..000000000 --- a/cicd/sync/.vagrant/machines/llb2/virtualbox/action_set_name +++ /dev/null @@ -1 +0,0 @@ -1691736942 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/box_meta b/cicd/sync/.vagrant/machines/llb2/virtualbox/box_meta deleted file mode 100644 index 4e311f257..000000000 --- a/cicd/sync/.vagrant/machines/llb2/virtualbox/box_meta +++ /dev/null @@ -1 +0,0 @@ -{"name":"sysnet4admin/Ubuntu-k8s","version":"0.7.1","provider":"virtualbox","directory":"boxes/sysnet4admin-VAGRANTSLASH-Ubuntu-k8s/0.7.1/virtualbox"} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/creator_uid b/cicd/sync/.vagrant/machines/llb2/virtualbox/creator_uid deleted file mode 100644 index e37d32abb..000000000 --- a/cicd/sync/.vagrant/machines/llb2/virtualbox/creator_uid +++ /dev/null @@ -1 +0,0 @@ -1000 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/id b/cicd/sync/.vagrant/machines/llb2/virtualbox/id deleted file mode 100644 index 94b1f98dd..000000000 --- a/cicd/sync/.vagrant/machines/llb2/virtualbox/id +++ /dev/null @@ -1 +0,0 @@ -381e6633-ff00-49ed-85ef-fd688e60e881 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/index_uuid b/cicd/sync/.vagrant/machines/llb2/virtualbox/index_uuid deleted file mode 100644 index 5a6f1a2b3..000000000 --- a/cicd/sync/.vagrant/machines/llb2/virtualbox/index_uuid +++ /dev/null @@ -1 +0,0 @@ -c29eda8a35d84899b80ff63c64775c03 \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/private_key b/cicd/sync/.vagrant/machines/llb2/virtualbox/private_key deleted file mode 100644 index 3de083641..000000000 --- a/cicd/sync/.vagrant/machines/llb2/virtualbox/private_key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEAuL4q7lqoaojZFfr9T+EGB9ovWb4yYgHYKneV/r6XLb5eo+8I -DWHbryKZ8LrQrpUcAHT6sjvPamt1bLTWMV220i3rV0IkyA51MyP9xQdJyiwrrcpq -Q7V32XHk56vZ6Tctdg+yoBnwN/jf4rTmcOE5LgyW+zY1EH752sTGvmsWFZZQVqnv -c4gdwVBCOvLEikOCOzauKL9tn8QYZ9FDHj5AmTd6OvM+fzoS2RwP6VB0gYGGKRom -KyX4f01UiKGkEpERP3j69sm+skp2m1aWrgEAzJShSgmTmS/Fjd8rkFusYh/tgQxE -Uml2r4aEE8klD/wc/vMBz+QG6vFoLpQdTYQY4wIDAQABAoIBACMcwbwbrvbQm7H5 -yjtsYWEMrwM3iwvlndg0cmXAg6tGC2bM/5LLjp1yzh9a4l7MKE5N7XgCtiFb2y2W -ET/AuAaqg5hKEgOT++KEbK8N+5BZZNPbLOdBjk3dv8vodbpFqnZN9X1IcMeynZrR -FxtNMzj1W5zTaLKB76sMy1lVsRTY8g6GWsmtuD8QksNVpa3voWc0SUXwVbraBYT9 -+oK5qx80Jm1bhXOVCjvlm/Zq74mHVfg1s+cv+Rv1CG9hluRERwNFjkIW4o2ykVXP -VhzL9Dy8A9RKGd9gz29buD5+NtEA9wiRucfXnKMpyhlo5q8fR5RcAOkV95J+Bj8B -y+SzIkECgYEA5ktZoJ89abWti3L059dYbfVCZYI/qQP9s5SDo0lwcsqodDrTneuN -4e1USl0k9fTTiiGzF+IxHbvoqpiyPI0+iBtGGzSkxL//k2/n831kp8BD0O+oD/EM -AFJvdSVQW/KD/XUfyPUPTKyQ4MJNB1dOcLjq2t/Bnkvz7icJLsHIuI0CgYEAzV0v -j8YkMJOX0yU1ajCW6Q4Zbr/zcYLz7iKlRQ7CApE+1fWPiefNxwQeGCrbTM2l8DM6 -Fb9sN2om6qCkyzPet65hiZw48KUTK1d7i8HqGL8In1mZ8pvq5+vK+SBuBVALk8aw -F4e+XhNE8a4nxHEIn4nc91EACBkJyvwIUt9G0y8CgYA4BeaU24guGiNQxDNt3wZF -IZ/4X7oq3KywLhXj+C6hzkkylUj7pt0KNBkISiZ+qwN3HAGccssuB5NvRNw/PMEU -jUX9wk2/C1IaROPxaPS+llgpENQapLIdtfAlWGF3LMUNOJv14B/LsENCWiIxAf00 -ENgZTMumPaXmrppeRIgW1QKBgCbVjU3p0SQidN5ntd8byx3NHcAUqkt30PriMFCL -xuJHZxN8uSUB4abVv38gc5HlfZCbj0tXhHZCK5PIxwDqER646hbkpNvUy3gxtdoq -1ELTtSS1A+id1WD/9MUBfeyhQ18xflEkyTnu+OWe4kRCf7fzkQZvmcruOSzaVhSd -xj3fAoGAfyHK6RgZuH59eyEKaSoCeXn+V+MVx407rxauCIMncHTEnysouWxalwX2 -mZW75xmOzwtvoXdQBlFpD1RozMAKHx7nDNavY04IYh7p4t66OKG7axP/P/8fpwGW -6z+DjGHJeQkEanEncs57mWHSNhqhBJI2amGyT0sYLwDaBzaX1SA= ------END RSA PRIVATE KEY----- diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/synced_folders b/cicd/sync/.vagrant/machines/llb2/virtualbox/synced_folders deleted file mode 100644 index 38126049a..000000000 --- a/cicd/sync/.vagrant/machines/llb2/virtualbox/synced_folders +++ /dev/null @@ -1 +0,0 @@ -{"virtualbox":{"/vagrant":{"guestpath":"/vagrant","hostpath":"/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync","disabled":false,"__vagrantfile":true}}} \ No newline at end of file diff --git a/cicd/sync/.vagrant/machines/llb2/virtualbox/vagrant_cwd b/cicd/sync/.vagrant/machines/llb2/virtualbox/vagrant_cwd deleted file mode 100644 index 7c7e228bb..000000000 --- a/cicd/sync/.vagrant/machines/llb2/virtualbox/vagrant_cwd +++ /dev/null @@ -1 +0,0 @@ -/home/netlox/loxilb-io/nik-netlox/loxilb/cicd/sync \ No newline at end of file diff --git a/cicd/sync/.vagrant/rgloader/loader.rb b/cicd/sync/.vagrant/rgloader/loader.rb deleted file mode 100644 index c3c05b095..000000000 --- a/cicd/sync/.vagrant/rgloader/loader.rb +++ /dev/null @@ -1,9 +0,0 @@ -# This file loads the proper rgloader/loader.rb file that comes packaged -# with Vagrant so that encoded files can properly run with Vagrant. - -if ENV["VAGRANT_INSTALLER_EMBEDDED_DIR"] - require File.expand_path( - "rgloader/loader", ENV["VAGRANT_INSTALLER_EMBEDDED_DIR"]) -else - raise "Encoded files can't be read outside of the Vagrant installer." -end From 627b9d58cae7920be39dffe48350a3ad83ed06b8 Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Fri, 18 Aug 2023 06:07:00 -0400 Subject: [PATCH 3/8] Initial commit for grpc xsync changes --- go.mod | 2 +- loxinet/dpbroker.go | 298 ++++++++++++++++++++++----------------- loxinet/xsync.pb.go | 298 +++++++++++++++++++++++++++++++++++++++ loxinet/xsync.proto | 24 ++++ loxinet/xsync_grpc.pb.go | 249 ++++++++++++++++++++++++++++++++ loxinet/xsync_server.go | 202 ++++++++++++++++++++++++++ 6 files changed, 940 insertions(+), 133 deletions(-) create mode 100644 loxinet/xsync.pb.go create mode 100644 loxinet/xsync.proto create mode 100644 loxinet/xsync_grpc.pb.go create mode 100644 loxinet/xsync_server.go diff --git a/go.mod b/go.mod index e4b9edd9c..86e2ed743 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/go-openapi/analysis v0.21.4 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.2 github.com/google/uuid v1.3.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/loxinet/dpbroker.go b/loxinet/dpbroker.go index 7a46170b2..947dd77aa 100644 --- a/loxinet/dpbroker.go +++ b/loxinet/dpbroker.go @@ -18,6 +18,7 @@ package loxinet import ( "bufio" + "context" "errors" "fmt" "io" @@ -30,6 +31,13 @@ import ( cmn "github.com/loxilb-io/loxilb/common" tk "github.com/loxilb-io/loxilib" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/types/known/anypb" + "github.com/golang/protobuf/ptypes/any" + "github.com/golang/protobuf/ptypes/wrappers" + "google.golang.org/protobuf/proto" + "encoding/json" ) // man names constants @@ -81,6 +89,7 @@ const ( const ( DpWorkQLen = 1024 XSyncPort = 22222 + XSyncPortGRPC = 22223 DpTiVal = 20 ) @@ -318,11 +327,17 @@ type DpCtInfo struct { BlockNum uint16 } +const ( + RPCTypeGoRPC = iota + RPCTypeGRPC +) + // XSync - Remote sync peer information type XSync struct { RemoteID int RPCState bool // For peer to peer RPC + RPCType int } // UlClDpWorkQ - work queue entry for ul-cl filter related operation @@ -401,10 +416,16 @@ type DpHookInterface interface { DpRelLock() } +type GRPCClient struct { + conn *grpc.ClientConn + xclient XSyncClient +} + // DpPeer - Remote DP Peer information type DpPeer struct { Peer net.IP - Client *rpc.Client + //Client *rpc.Client + Client interface{} } // DpH - datapath context container @@ -452,21 +473,48 @@ func dialHTTPPath(network, address, path string) (*rpc.Client, error) { // DpXsyncRPCReset - Routine to reset Sunc RPC Client connections func (dp *DpH) DpXsyncRPCReset() int { + var err error dp.SyncMtx.Lock() defer dp.SyncMtx.Unlock() for idx := range mh.dp.Peers { pe := &mh.dp.Peers[idx] - if pe.Client != nil { - pe.Client.Close() - pe.Client = nil - } - if pe.Client == nil { + if dp.RPC.RPCType == RPCTypeGoRPC { cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) - pe.Client, _ = dialHTTPPath("tcp", cStr, rpc.DefaultRPCPath) + client, ok := pe.Client.(*rpc.Client) + if ok && client != nil { + client.Close() + pe.Client = nil + } + if pe.Client == nil { + pe.Client, err = dialHTTPPath("tcp", cStr, rpc.DefaultRPCPath) + if pe.Client == nil { + tk.LogIt(tk.LogInfo,"Failed to dial xsync goRPC pair: %v", err) + return -1 + } + tk.LogIt(tk.LogInfo, "XSync goRPC - %s :Reset\n", cStr) + } + } else { + var opts []grpc.DialOption + var cinfo GRPCClient + client, ok := pe.Client.(GRPCClient) + if ok { + client.conn.Close() + pe.Client = nil + } + if pe.Client == nil { - return -1 + cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPortGRPC) + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + cinfo.conn, err = grpc.Dial(cStr, opts...) + if err != nil { + tk.LogIt(tk.LogInfo,"Failed to dial xsync gRPC pair: %v", err) + return -1 + } + + cinfo.xclient = NewXSyncClient(cinfo.conn) + pe.Client = cinfo + tk.LogIt(tk.LogInfo, "XSync gRPC - %s :Reset\n", cStr) } - tk.LogIt(tk.LogInfo, "XSync RPC - %s :Reset\n", cStr) } } return 0 @@ -477,10 +525,7 @@ func (dp *DpH) DpXsyncInSync() bool { dp.SyncMtx.Lock() defer dp.SyncMtx.Unlock() - if len(dp.Remotes) >= len(mh.has.NodeMap) { - return true - } - return false + return len(dp.Remotes) >= len(mh.has.NodeMap) } // WaitXsyncReady - Routine to wait till it ready for syncing the peer entity @@ -498,9 +543,46 @@ func (dp *DpH) WaitXsyncReady(who string) { } } +func callGRPC(client XSyncClient, rpcCallStr string, args interface{}, reply *int) error { + var err error + var xreply *XSyncReply + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + anyValue := &any.Any{} + + if rpcCallStr != "XSync.DpWorkOnCtGet" { + bytes, _ := json.Marshal(args) + bytesValue := &wrappers.BytesValue{ + Value: bytes, + } + err = anypb.MarshalFrom(anyValue, bytesValue, proto.MarshalOptions{}) + } + + if rpcCallStr == "XSync.DpWorkOnBlockCtAdd" { + xreply, err = client.DpWorkOnBlockCtAddGRPC(ctx, &ConnInfo{Cti: anyValue}) + } else if rpcCallStr == "XSync.DpWorkOnBlockCtDel" { + xreply, err = client.DpWorkOnBlockCtDelGRPC(ctx, &ConnInfo{Cti: anyValue}) + } else if rpcCallStr == "XSync.DpWorkOnCtAdd" { + xreply, err = client.DpWorkOnCtAddGRPC(ctx, &ConnInfo{Cti: anyValue}) + } else if rpcCallStr == "XSync.DpWorkOnCtDelete" { + xreply, err = client.DpWorkOnCtDelGRPC(ctx, &ConnInfo{Cti: anyValue}) + } else if rpcCallStr == "XSync.DpWorkOnCtGet" { + xreply, err = client.DpWorkOnCtGetGRPC(ctx, &ConnGet{Async: args.(int32)}) + } + if (err != nil) { + tk.LogIt(tk.LogInfo, "XSync %s reply - %v\n", rpcCallStr, err.Error()) + } else if xreply != nil { + *reply = int(xreply.Response) + tk.LogIt(tk.LogInfo, "XSync %s reply - %v[OK]\n", rpcCallStr, *reply) + } + return err +} + // DpXsyncRPC - Routine for syncing connection information with peers func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { var reply int + var err error + timeout := 2 * time.Second dp.SyncMtx.Lock() defer dp.SyncMtx.Unlock() @@ -525,15 +607,32 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { restartRPC: pe := &mh.dp.Peers[idx] if pe.Client == nil { - cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) var err error - pe.Client, err = dialHTTPPath("tcp", cStr, rpc.DefaultRPCPath) - if pe.Client == nil { - tk.LogIt(tk.LogInfo, "XSync RPC - %s :Fail(%s)\n", cStr, err) - rpcErr = true - continue + if dp.RPC.RPCType == RPCTypeGoRPC { + cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) + pe.Client, err = dialHTTPPath("tcp", cStr, rpc.DefaultRPCPath) + if pe.Client == nil { + tk.LogIt(tk.LogInfo, "XSync RPC - %s :Fail(%s)\n", cStr, err) + rpcErr = true + continue + } + tk.LogIt(tk.LogInfo, "XSync goRPC - %s :Connected\n", cStr) + } else { + var opts []grpc.DialOption + var cinfo GRPCClient + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPortGRPC) + cinfo.conn, err = grpc.Dial(cStr, opts...) + if err != nil { + tk.LogIt(tk.LogInfo,"Failed to dial xsync gRPC pair: %v", err) + return -1 + } + + cinfo.xclient = NewXSyncClient(cinfo.conn) + pe.Client = cinfo + tk.LogIt(tk.LogInfo, "XSync gRPC - %s :Connected\n", cStr) } - tk.LogIt(tk.LogInfo, "XSync RPC - %s :Connected\n", cStr) + } reply = 0 @@ -575,20 +674,45 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { return 0 } } - if cti != nil { - call = pe.Client.Go(rpcCallStr, *cti, &reply, make(chan *rpc.Call, 1)) + if dp.RPC.RPCType == RPCTypeGoRPC { + client, _ := pe.Client.(*rpc.Client) + if cti != nil { + call = client.Go(rpcCallStr, *cti, &reply, make(chan *rpc.Call, 1)) + } else { + call = client.Go(rpcCallStr, blkCti, &reply, make(chan *rpc.Call, 1)) + } } else { - call = pe.Client.Go(rpcCallStr, blkCti, &reply, make(chan *rpc.Call, 1)) + client, _ := pe.Client.(GRPCClient) + if cti != nil { + err = callGRPC(client.xclient, rpcCallStr, *cti, &reply) + } else { + err = callGRPC(client.xclient, rpcCallStr, blkCti, &reply) + } } } else { async := 1 - call = pe.Client.Go(rpcCallStr, async, &reply, make(chan *rpc.Call, 1)) + if dp.RPC.RPCType == RPCTypeGoRPC { + call = pe.Client.(*rpc.Client).Go(rpcCallStr, async, &reply, make(chan *rpc.Call, 1)) + } else { + err = callGRPC(pe.Client.(GRPCClient).xclient, rpcCallStr, int32(async), &reply) + } + } + + if dp.RPC.RPCType == RPCTypeGoRPC { + resp := <-call.Done + if resp != nil { + err = resp.Error + } } select { case <-time.After(timeout): tk.LogIt(tk.LogError, "rpc call timeout(%v)\n", timeout) if pe.Client != nil { - pe.Client.Close() + if dp.RPC.RPCType == RPCTypeGoRPC { + pe.Client.(*rpc.Client).Close() + } else { + pe.Client.(GRPCClient).conn.Close() + } } pe.Client = nil rpcRetries++ @@ -596,14 +720,13 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { goto restartRPC } rpcErr = true - case resp := <-call.Done: - if resp != nil && resp.Error != nil { - tk.LogIt(tk.LogError, "rpc call failed(%s)\n", resp.Error) - rpcErr = true - } - if reply != 0 { - rpcErr = true - } + } + if err != nil { + tk.LogIt(tk.LogError, "rpc call failed(%s)\n", err) + rpcErr = true + } + if reply != 0 { + rpcErr = true } } @@ -622,110 +745,17 @@ func DpBrokerInit(dph DpHookInterface) *DpH { nDp.ToFinCh = make(chan int) nDp.DpHooks = dph nDp.RPC = new(XSync) + + nDp.RPC.RPCType = RPCTypeGRPC + if nDp.RPC.RPCType == RPCTypeGRPC { + go startxSyncGRPCServer() + } go DpWorker(nDp, nDp.ToFinCh, nDp.ToDpCh) return nDp } -// DpWorkOnBlockCtAdd - Add block CT entries from remote -func (xs *XSync) DpWorkOnBlockCtAdd(blockCtis []DpCtInfo, ret *int) error { - if !mh.ready { - return errors.New("Not-Ready") - } - - *ret = 0 - for _, cti := range blockCtis { - - tk.LogIt(tk.LogDebug, "RPC - Block CT Add %s\n", cti.Key()) - r := mh.dp.DpHooks.DpCtAdd(&cti) - if r != 0 { - *ret = r - } - } - - return nil -} - -// DpWorkOnCtAdd - Add a CT entry from remote -func (xs *XSync) DpWorkOnCtAdd(cti DpCtInfo, ret *int) error { - if !mh.ready { - return errors.New("Not-Ready") - } - - if cti.Proto == "xsync" { - mh.dp.SyncMtx.Lock() - defer mh.dp.SyncMtx.Unlock() - - for idx := range mh.dp.Remotes { - r := &mh.dp.Remotes[idx] - if r.RemoteID == int(cti.Sport) { - r.RPCState = true - *ret = 0 - return nil - } - } - - r := XSync{RemoteID: int(cti.Sport), RPCState: true} - mh.dp.Remotes = append(mh.dp.Remotes, r) - - tk.LogIt(tk.LogDebug, "RPC - CT Xsync Remote-%v\n", cti.Sport) - - *ret = 0 - return nil - } - - tk.LogIt(tk.LogDebug, "RPC - CT Add %s\n", cti.Key()) - r := mh.dp.DpHooks.DpCtAdd(&cti) - *ret = r - return nil -} - -// DpWorkOnBlockCtDelete - Add block CT entries from remote -func (xs *XSync) DpWorkOnBlockCtDelete(blockCtis []DpCtInfo, ret *int) error { - if !mh.ready { - return errors.New("Not-Ready") - } - - *ret = 0 - for _, cti := range blockCtis { - - tk.LogIt(tk.LogDebug, "RPC - Block CT Del %s\n", cti.Key()) - r := mh.dp.DpHooks.DpCtDel(&cti) - if r != 0 { - *ret = r - } - } - - return nil -} - -// DpWorkOnCtDelete - Delete a CT entry from remote -func (xs *XSync) DpWorkOnCtDelete(cti DpCtInfo, ret *int) error { - if !mh.ready { - return errors.New("Not-Ready") - } - tk.LogIt(tk.LogDebug, "RPC - CT Del %s\n", cti.Key()) - r := mh.dp.DpHooks.DpCtDel(&cti) - *ret = r - return nil -} - -// DpWorkOnCtGet - Get all CT entries asynchronously -func (xs *XSync) DpWorkOnCtGet(async int, ret *int) error { - if !mh.ready { - return errors.New("Not-Ready") - } - - // Most likely need to reset reverse rpc channel - mh.dp.DpXsyncRPCReset() - - tk.LogIt(tk.LogDebug, "RPC - CT Get %d\n", async) - mh.dp.DpHooks.DpCtGetAsync() - *ret = 0 - - return nil -} // DpWorkOnPort - routine to work on a port work queue request func (dp *DpH) DpWorkOnPort(pWq *PortDpWorkQ) DpRetT { @@ -865,7 +895,11 @@ func (dp *DpH) DpWorkOnPeerOp(pWq *PeerDpWorkQ) DpRetT { pe := &dp.Peers[idx] if pe.Peer.Equal(pWq.PeerIP) { if pe.Client != nil { - pe.Client.Close() + if dp.RPC.RPCType == RPCTypeGoRPC { + pe.Client.(*rpc.Client).Close() + } else { + pe.Client.(GRPCClient).conn.Close() + } } dp.Peers = append(dp.Peers[:idx], dp.Peers[idx+1:]...) tk.LogIt(tk.LogInfo, "Deleted cluster-peer %s\n", pWq.PeerIP.String()) diff --git a/loxinet/xsync.pb.go b/loxinet/xsync.pb.go new file mode 100644 index 000000000..cfd336d76 --- /dev/null +++ b/loxinet/xsync.pb.go @@ -0,0 +1,298 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.6.1 +// source: loxinet/xsync.proto + +package loxinet + +import ( + any1 "github.com/golang/protobuf/ptypes/any" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type XSyncReply struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Response int32 `protobuf:"varint,1,opt,name=response,proto3" json:"response,omitempty"` +} + +func (x *XSyncReply) Reset() { + *x = XSyncReply{} + if protoimpl.UnsafeEnabled { + mi := &file_loxinet_xsync_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *XSyncReply) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*XSyncReply) ProtoMessage() {} + +func (x *XSyncReply) ProtoReflect() protoreflect.Message { + mi := &file_loxinet_xsync_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use XSyncReply.ProtoReflect.Descriptor instead. +func (*XSyncReply) Descriptor() ([]byte, []int) { + return file_loxinet_xsync_proto_rawDescGZIP(), []int{0} +} + +func (x *XSyncReply) GetResponse() int32 { + if x != nil { + return x.Response + } + return 0 +} + +type ConnInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Cti *any1.Any `protobuf:"bytes,1,opt,name=cti,proto3" json:"cti,omitempty"` +} + +func (x *ConnInfo) Reset() { + *x = ConnInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_loxinet_xsync_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ConnInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConnInfo) ProtoMessage() {} + +func (x *ConnInfo) ProtoReflect() protoreflect.Message { + mi := &file_loxinet_xsync_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConnInfo.ProtoReflect.Descriptor instead. +func (*ConnInfo) Descriptor() ([]byte, []int) { + return file_loxinet_xsync_proto_rawDescGZIP(), []int{1} +} + +func (x *ConnInfo) GetCti() *any1.Any { + if x != nil { + return x.Cti + } + return nil +} + +type ConnGet struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Async int32 `protobuf:"varint,1,opt,name=async,proto3" json:"async,omitempty"` +} + +func (x *ConnGet) Reset() { + *x = ConnGet{} + if protoimpl.UnsafeEnabled { + mi := &file_loxinet_xsync_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ConnGet) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConnGet) ProtoMessage() {} + +func (x *ConnGet) ProtoReflect() protoreflect.Message { + mi := &file_loxinet_xsync_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConnGet.ProtoReflect.Descriptor instead. +func (*ConnGet) Descriptor() ([]byte, []int) { + return file_loxinet_xsync_proto_rawDescGZIP(), []int{2} +} + +func (x *ConnGet) GetAsync() int32 { + if x != nil { + return x.Async + } + return 0 +} + +var File_loxinet_xsync_proto protoreflect.FileDescriptor + +var file_loxinet_xsync_proto_rawDesc = []byte{ + 0x0a, 0x13, 0x6c, 0x6f, 0x78, 0x69, 0x6e, 0x65, 0x74, 0x2f, 0x78, 0x73, 0x79, 0x6e, 0x63, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x22, 0x28, 0x0a, 0x0a, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x1a, + 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x32, 0x0a, 0x08, 0x43, 0x6f, + 0x6e, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x26, 0x0a, 0x03, 0x63, 0x74, 0x69, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x03, 0x63, 0x74, 0x69, 0x22, 0x1f, + 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x47, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x61, 0x73, 0x79, + 0x6e, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x32, + 0xfb, 0x01, 0x0a, 0x05, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x32, 0x0a, 0x16, 0x44, 0x70, 0x57, + 0x6f, 0x72, 0x6b, 0x4f, 0x6e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x43, 0x74, 0x41, 0x64, 0x64, 0x47, + 0x52, 0x50, 0x43, 0x12, 0x09, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x0b, + 0x2e, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x2d, 0x0a, + 0x11, 0x44, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x4f, 0x6e, 0x43, 0x74, 0x41, 0x64, 0x64, 0x47, 0x52, + 0x50, 0x43, 0x12, 0x09, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x0b, 0x2e, + 0x58, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x32, 0x0a, 0x16, + 0x44, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x4f, 0x6e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x43, 0x74, 0x44, + 0x65, 0x6c, 0x47, 0x52, 0x50, 0x43, 0x12, 0x09, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x49, 0x6e, 0x66, + 0x6f, 0x1a, 0x0b, 0x2e, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, + 0x12, 0x2d, 0x0a, 0x11, 0x44, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x4f, 0x6e, 0x43, 0x74, 0x44, 0x65, + 0x6c, 0x47, 0x52, 0x50, 0x43, 0x12, 0x09, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x49, 0x6e, 0x66, 0x6f, + 0x1a, 0x0b, 0x2e, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, + 0x2c, 0x0a, 0x11, 0x44, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x4f, 0x6e, 0x43, 0x74, 0x47, 0x65, 0x74, + 0x47, 0x52, 0x50, 0x43, 0x12, 0x08, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x47, 0x65, 0x74, 0x1a, 0x0b, + 0x2e, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0b, 0x5a, + 0x09, 0x2e, 0x2f, 0x6c, 0x6f, 0x78, 0x69, 0x6e, 0x65, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, +} + +var ( + file_loxinet_xsync_proto_rawDescOnce sync.Once + file_loxinet_xsync_proto_rawDescData = file_loxinet_xsync_proto_rawDesc +) + +func file_loxinet_xsync_proto_rawDescGZIP() []byte { + file_loxinet_xsync_proto_rawDescOnce.Do(func() { + file_loxinet_xsync_proto_rawDescData = protoimpl.X.CompressGZIP(file_loxinet_xsync_proto_rawDescData) + }) + return file_loxinet_xsync_proto_rawDescData +} + +var file_loxinet_xsync_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_loxinet_xsync_proto_goTypes = []interface{}{ + (*XSyncReply)(nil), // 0: XSyncReply + (*ConnInfo)(nil), // 1: ConnInfo + (*ConnGet)(nil), // 2: ConnGet + (*any1.Any)(nil), // 3: google.protobuf.Any +} +var file_loxinet_xsync_proto_depIdxs = []int32{ + 3, // 0: ConnInfo.cti:type_name -> google.protobuf.Any + 1, // 1: XSync.DpWorkOnBlockCtAddGRPC:input_type -> ConnInfo + 1, // 2: XSync.DpWorkOnCtAddGRPC:input_type -> ConnInfo + 1, // 3: XSync.DpWorkOnBlockCtDelGRPC:input_type -> ConnInfo + 1, // 4: XSync.DpWorkOnCtDelGRPC:input_type -> ConnInfo + 2, // 5: XSync.DpWorkOnCtGetGRPC:input_type -> ConnGet + 0, // 6: XSync.DpWorkOnBlockCtAddGRPC:output_type -> XSyncReply + 0, // 7: XSync.DpWorkOnCtAddGRPC:output_type -> XSyncReply + 0, // 8: XSync.DpWorkOnBlockCtDelGRPC:output_type -> XSyncReply + 0, // 9: XSync.DpWorkOnCtDelGRPC:output_type -> XSyncReply + 0, // 10: XSync.DpWorkOnCtGetGRPC:output_type -> XSyncReply + 6, // [6:11] is the sub-list for method output_type + 1, // [1:6] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_loxinet_xsync_proto_init() } +func file_loxinet_xsync_proto_init() { + if File_loxinet_xsync_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_loxinet_xsync_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*XSyncReply); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_loxinet_xsync_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ConnInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_loxinet_xsync_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ConnGet); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_loxinet_xsync_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_loxinet_xsync_proto_goTypes, + DependencyIndexes: file_loxinet_xsync_proto_depIdxs, + MessageInfos: file_loxinet_xsync_proto_msgTypes, + }.Build() + File_loxinet_xsync_proto = out.File + file_loxinet_xsync_proto_rawDesc = nil + file_loxinet_xsync_proto_goTypes = nil + file_loxinet_xsync_proto_depIdxs = nil +} diff --git a/loxinet/xsync.proto b/loxinet/xsync.proto new file mode 100644 index 000000000..8647ee8e4 --- /dev/null +++ b/loxinet/xsync.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; +import "google/protobuf/any.proto"; +option go_package = "./loxinet"; + +message XSyncReply { + int32 response = 1; +} + +message ConnInfo { + google.protobuf.Any cti = 1; +} + +message ConnGet { + int32 async = 1; +} +// The xsync service definition. +service XSync { + rpc DpWorkOnBlockCtAddGRPC (ConnInfo) returns (XSyncReply) {} + rpc DpWorkOnCtAddGRPC (ConnInfo) returns (XSyncReply) {} + rpc DpWorkOnBlockCtDelGRPC (ConnInfo) returns (XSyncReply) {} + rpc DpWorkOnCtDelGRPC (ConnInfo) returns (XSyncReply) {} + rpc DpWorkOnCtGetGRPC (ConnGet) returns (XSyncReply) {} + } + \ No newline at end of file diff --git a/loxinet/xsync_grpc.pb.go b/loxinet/xsync_grpc.pb.go new file mode 100644 index 000000000..a83e21601 --- /dev/null +++ b/loxinet/xsync_grpc.pb.go @@ -0,0 +1,249 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.6.1 +// source: loxinet/xsync.proto + +package loxinet + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// XSyncClient is the client API for XSync service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type XSyncClient interface { + DpWorkOnBlockCtAddGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) + DpWorkOnCtAddGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) + DpWorkOnBlockCtDelGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) + DpWorkOnCtDelGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) + DpWorkOnCtGetGRPC(ctx context.Context, in *ConnGet, opts ...grpc.CallOption) (*XSyncReply, error) +} + +type xSyncClient struct { + cc grpc.ClientConnInterface +} + +func NewXSyncClient(cc grpc.ClientConnInterface) XSyncClient { + return &xSyncClient{cc} +} + +func (c *xSyncClient) DpWorkOnBlockCtAddGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) { + out := new(XSyncReply) + err := c.cc.Invoke(ctx, "/XSync/DpWorkOnBlockCtAddGRPC", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *xSyncClient) DpWorkOnCtAddGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) { + out := new(XSyncReply) + err := c.cc.Invoke(ctx, "/XSync/DpWorkOnCtAddGRPC", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *xSyncClient) DpWorkOnBlockCtDelGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) { + out := new(XSyncReply) + err := c.cc.Invoke(ctx, "/XSync/DpWorkOnBlockCtDelGRPC", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *xSyncClient) DpWorkOnCtDelGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) { + out := new(XSyncReply) + err := c.cc.Invoke(ctx, "/XSync/DpWorkOnCtDelGRPC", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *xSyncClient) DpWorkOnCtGetGRPC(ctx context.Context, in *ConnGet, opts ...grpc.CallOption) (*XSyncReply, error) { + out := new(XSyncReply) + err := c.cc.Invoke(ctx, "/XSync/DpWorkOnCtGetGRPC", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// XSyncServer is the server API for XSync service. +// All implementations must embed UnimplementedXSyncServer +// for forward compatibility +type XSyncServer interface { + DpWorkOnBlockCtAddGRPC(context.Context, *ConnInfo) (*XSyncReply, error) + DpWorkOnCtAddGRPC(context.Context, *ConnInfo) (*XSyncReply, error) + DpWorkOnBlockCtDelGRPC(context.Context, *ConnInfo) (*XSyncReply, error) + DpWorkOnCtDelGRPC(context.Context, *ConnInfo) (*XSyncReply, error) + DpWorkOnCtGetGRPC(context.Context, *ConnGet) (*XSyncReply, error) + mustEmbedUnimplementedXSyncServer() +} + +// UnimplementedXSyncServer must be embedded to have forward compatible implementations. +type UnimplementedXSyncServer struct { +} + +func (UnimplementedXSyncServer) DpWorkOnBlockCtAddGRPC(context.Context, *ConnInfo) (*XSyncReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method DpWorkOnBlockCtAddGRPC not implemented") +} +func (UnimplementedXSyncServer) DpWorkOnCtAddGRPC(context.Context, *ConnInfo) (*XSyncReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method DpWorkOnCtAddGRPC not implemented") +} +func (UnimplementedXSyncServer) DpWorkOnBlockCtDelGRPC(context.Context, *ConnInfo) (*XSyncReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method DpWorkOnBlockCtDelGRPC not implemented") +} +func (UnimplementedXSyncServer) DpWorkOnCtDelGRPC(context.Context, *ConnInfo) (*XSyncReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method DpWorkOnCtDelGRPC not implemented") +} +func (UnimplementedXSyncServer) DpWorkOnCtGetGRPC(context.Context, *ConnGet) (*XSyncReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method DpWorkOnCtGetGRPC not implemented") +} +func (UnimplementedXSyncServer) mustEmbedUnimplementedXSyncServer() {} + +// UnsafeXSyncServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to XSyncServer will +// result in compilation errors. +type UnsafeXSyncServer interface { + mustEmbedUnimplementedXSyncServer() +} + +func RegisterXSyncServer(s grpc.ServiceRegistrar, srv XSyncServer) { + s.RegisterService(&XSync_ServiceDesc, srv) +} + +func _XSync_DpWorkOnBlockCtAddGRPC_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ConnInfo) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(XSyncServer).DpWorkOnBlockCtAddGRPC(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/XSync/DpWorkOnBlockCtAddGRPC", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(XSyncServer).DpWorkOnBlockCtAddGRPC(ctx, req.(*ConnInfo)) + } + return interceptor(ctx, in, info, handler) +} + +func _XSync_DpWorkOnCtAddGRPC_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ConnInfo) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(XSyncServer).DpWorkOnCtAddGRPC(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/XSync/DpWorkOnCtAddGRPC", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(XSyncServer).DpWorkOnCtAddGRPC(ctx, req.(*ConnInfo)) + } + return interceptor(ctx, in, info, handler) +} + +func _XSync_DpWorkOnBlockCtDelGRPC_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ConnInfo) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(XSyncServer).DpWorkOnBlockCtDelGRPC(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/XSync/DpWorkOnBlockCtDelGRPC", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(XSyncServer).DpWorkOnBlockCtDelGRPC(ctx, req.(*ConnInfo)) + } + return interceptor(ctx, in, info, handler) +} + +func _XSync_DpWorkOnCtDelGRPC_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ConnInfo) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(XSyncServer).DpWorkOnCtDelGRPC(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/XSync/DpWorkOnCtDelGRPC", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(XSyncServer).DpWorkOnCtDelGRPC(ctx, req.(*ConnInfo)) + } + return interceptor(ctx, in, info, handler) +} + +func _XSync_DpWorkOnCtGetGRPC_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ConnGet) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(XSyncServer).DpWorkOnCtGetGRPC(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/XSync/DpWorkOnCtGetGRPC", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(XSyncServer).DpWorkOnCtGetGRPC(ctx, req.(*ConnGet)) + } + return interceptor(ctx, in, info, handler) +} + +// XSync_ServiceDesc is the grpc.ServiceDesc for XSync service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var XSync_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "XSync", + HandlerType: (*XSyncServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "DpWorkOnBlockCtAddGRPC", + Handler: _XSync_DpWorkOnBlockCtAddGRPC_Handler, + }, + { + MethodName: "DpWorkOnCtAddGRPC", + Handler: _XSync_DpWorkOnCtAddGRPC_Handler, + }, + { + MethodName: "DpWorkOnBlockCtDelGRPC", + Handler: _XSync_DpWorkOnBlockCtDelGRPC_Handler, + }, + { + MethodName: "DpWorkOnCtDelGRPC", + Handler: _XSync_DpWorkOnCtDelGRPC_Handler, + }, + { + MethodName: "DpWorkOnCtGetGRPC", + Handler: _XSync_DpWorkOnCtGetGRPC_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "loxinet/xsync.proto", +} diff --git a/loxinet/xsync_server.go b/loxinet/xsync_server.go new file mode 100644 index 000000000..ed7d97732 --- /dev/null +++ b/loxinet/xsync_server.go @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2022 NetLOX Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loxinet + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + + "github.com/golang/protobuf/ptypes/wrappers" + //"github.com/loxilb-io/loxilb/loxinet" + tk "github.com/loxilb-io/loxilib" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +// DpWorkOnBlockCtAdd - Add block CT entries from remote goRPC client +func (xs *XSync) DpWorkOnBlockCtAdd(blockCtis []DpCtInfo, ret *int) error { + if !mh.ready { + return errors.New("Not-Ready") + } + + *ret = 0 + + mh.dp.DpHooks.DpGetLock() + + for _, cti := range blockCtis { + + tk.LogIt(tk.LogDebug, "RPC - Block CT Add %s\n", cti.Key()) + r := mh.dp.DpHooks.DpCtAdd(&cti) + if r != 0 { + *ret = r + } + } + + mh.dp.DpHooks.DpRelLock() + + return nil +} + +// DpWorkOnCtAdd - Add a CT entry from remote goRPC client +func (xs *XSync) DpWorkOnCtAdd(cti DpCtInfo, ret *int) error { + if !mh.ready { + return errors.New("Not-Ready") + } + + if cti.Proto == "xsync" { + mh.dp.SyncMtx.Lock() + defer mh.dp.SyncMtx.Unlock() + + for idx := range mh.dp.Remotes { + r := &mh.dp.Remotes[idx] + if r.RemoteID == int(cti.Sport) { + r.RPCState = true + *ret = 0 + return nil + } + } + + r := XSync{RemoteID: int(cti.Sport), RPCState: true} + mh.dp.Remotes = append(mh.dp.Remotes, r) + + tk.LogIt(tk.LogDebug, "RPC - CT Xsync Remote-%v\n", cti.Sport) + + *ret = 0 + return nil + } + + tk.LogIt(tk.LogDebug, "RPC - CT Add %s\n", cti.Key()) + r := mh.dp.DpHooks.DpCtAdd(&cti) + *ret = r + return nil +} + +// DpWorkOnCtDelete - Delete a CT entry from remote goRPC client +func (xs *XSync) DpWorkOnCtDelete(cti DpCtInfo, ret *int) error { + if !mh.ready { + return errors.New("Not-Ready") + } + tk.LogIt(tk.LogDebug, "RPC - CT Del %s\n", cti.Key()) + r := mh.dp.DpHooks.DpCtDel(&cti) + *ret = r + return nil +} + +// DpWorkOnCtGet - Get all CT entries asynchronously goRPC client +func (xs *XSync) DpWorkOnCtGet(async int, ret *int) error { + if !mh.ready { + return errors.New("Not-Ready") + } + + // Most likely need to reset reverse rpc channel + mh.dp.DpXsyncRPCReset() + + tk.LogIt(tk.LogDebug, "RPC - CT Get %d\n", async) + mh.dp.DpHooks.DpCtGetAsync() + *ret = 0 + + return nil +} + +func (xs *XSync) DpWorkOnBlockCtAddGRPC(ctx context.Context, m *ConnInfo) (*XSyncReply, error) { + var value interface{} + var resp int + bytesValue := &wrappers.BytesValue{} + anyValue := m.Cti + + err := anypb.UnmarshalTo(anyValue, bytesValue, proto.UnmarshalOptions{}) + if err != nil { + return &XSyncReply{Response: -1}, err + } else { + uErr := json.Unmarshal(bytesValue.Value, &value) + if uErr != nil { + return &XSyncReply{Response: -1}, uErr + } + } + xs.DpWorkOnBlockCtAdd(value.([]DpCtInfo), &resp) + return &XSyncReply{Response: int32(resp)}, nil +} + +func (xs *XSync) DpWorkOnCtAddGRPC(ctx context.Context, m *ConnInfo) (*XSyncReply, error) { + var value interface{} + var resp int + bytesValue := &wrappers.BytesValue{} + anyValue := m.Cti + + err := anypb.UnmarshalTo(anyValue, bytesValue, proto.UnmarshalOptions{}) + if err != nil { + return &XSyncReply{Response: -1}, err + } else { + uErr := json.Unmarshal(bytesValue.Value, &value) + if uErr != nil { + return &XSyncReply{Response: -1}, uErr + } + } + + xs.DpWorkOnCtAdd(value.(DpCtInfo), &resp) + return &XSyncReply{Response: int32(resp)}, nil +} + +func (xs *XSync) DpWorkOnBlockCtDelGRPC(context.Context, *ConnInfo) (*XSyncReply, error) { + return nil, errors.New("method DpWorkOnBlockCtDelGRPC not implemented") +} + +func (xs *XSync) DpWorkOnCtDelGRPC(ctx context.Context, m *ConnInfo) (*XSyncReply, error) { + var value interface{} + var resp int + bytesValue := &wrappers.BytesValue{} + anyValue := m.Cti + + err := anypb.UnmarshalTo(anyValue, bytesValue, proto.UnmarshalOptions{}) + if err != nil { + return &XSyncReply{Response: -1}, err + } else { + uErr := json.Unmarshal(bytesValue.Value, &value) + if uErr != nil { + return &XSyncReply{Response: -1}, uErr + } + } + xs.DpWorkOnCtDelete(value.(DpCtInfo), &resp) + return &XSyncReply{Response: int32(resp)}, nil +} + +func (xs *XSync) DpWorkOnCtGetGRPC(ctx context.Context, m *ConnGet) (*XSyncReply, error) { + + var resp int + err := xs.DpWorkOnCtGet(int(m.Async), &resp) + + return &XSyncReply{Response: int32(resp)}, err +} + +func (xs *XSync) mustEmbedUnimplementedXSyncServer() {} + +func startxSyncGRPCServer() { + lis, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", XSyncPortGRPC)) + if err != nil { + tk.LogIt(tk.LogEmerg, "gRPC - Server Start Error\n") + return + } + grpcServer := grpc.NewServer() + s := XSync{} + RegisterXSyncServer(grpcServer, &s) + tk.LogIt(tk.LogNotice, "*******************gRPC - Server Started*****************\n") + grpcServer.Serve(lis) +} \ No newline at end of file From 2ee5812120b1940c99909153da64e2d3d510daf0 Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Wed, 23 Aug 2023 18:25:36 +0900 Subject: [PATCH 4/8] grpc sync changes --- loxinet/dpbroker.go | 106 +++++++++++++++++++++------------------- loxinet/xsync_server.go | 89 ++++++++++++++++++++++++++++++--- 2 files changed, 137 insertions(+), 58 deletions(-) diff --git a/loxinet/dpbroker.go b/loxinet/dpbroker.go index 947dd77aa..f38db123d 100644 --- a/loxinet/dpbroker.go +++ b/loxinet/dpbroker.go @@ -89,7 +89,7 @@ const ( const ( DpWorkQLen = 1024 XSyncPort = 22222 - XSyncPortGRPC = 22223 + XSyncPortGRPC = 22224 DpTiVal = 20 ) @@ -303,28 +303,28 @@ type NatDpWorkQ struct { // DpCtInfo - representation of a datapath conntrack information type DpCtInfo struct { - DIP net.IP - SIP net.IP - Dport uint16 - Sport uint16 - Proto string - CState string - CAct string - CI string - Packets uint64 - Bytes uint64 - Deleted int - PKey []byte - PVal []byte - LTs time.Time - NTs time.Time - XSync bool + DIP net.IP `json:"dip"` + SIP net.IP `json:"sip"` + Dport uint16 `json:"dport"` + Sport uint16 `json:"sport"` + Proto string `json:"proto"` + CState string `json:"cstate"` + CAct string `json:"cact"` + CI string `json:"ci"` + Packets uint64 `json:"packets"` + Bytes uint64 `json:"bytes"` + Deleted int `json:"deleted"` + PKey []byte `json:"pkey"` + PVal []byte `json:"pval"` + LTs time.Time `json:"lts"` + NTs time.Time `json:"nts"` + XSync bool `json:"xsync"` // LB Association Data - ServiceIP net.IP - ServProto string - L4ServPort uint16 - BlockNum uint16 + ServiceIP net.IP `json:"serviceip"` + ServProto string `json:"servproto"` + L4ServPort uint16 `json:"l4servproto"` + BlockNum uint16 `json:"blocknum"` } const ( @@ -560,7 +560,7 @@ func callGRPC(client XSyncClient, rpcCallStr string, args interface{}, reply *in if rpcCallStr == "XSync.DpWorkOnBlockCtAdd" { xreply, err = client.DpWorkOnBlockCtAddGRPC(ctx, &ConnInfo{Cti: anyValue}) - } else if rpcCallStr == "XSync.DpWorkOnBlockCtDel" { + } else if rpcCallStr == "XSync.DpWorkOnBlockCtDelete" { xreply, err = client.DpWorkOnBlockCtDelGRPC(ctx, &ConnInfo{Cti: anyValue}) } else if rpcCallStr == "XSync.DpWorkOnCtAdd" { xreply, err = client.DpWorkOnCtAddGRPC(ctx, &ConnInfo{Cti: anyValue}) @@ -569,7 +569,9 @@ func callGRPC(client XSyncClient, rpcCallStr string, args interface{}, reply *in } else if rpcCallStr == "XSync.DpWorkOnCtGet" { xreply, err = client.DpWorkOnCtGetGRPC(ctx, &ConnGet{Async: args.(int32)}) } + if (err != nil) { + *reply = -1 tk.LogIt(tk.LogInfo, "XSync %s reply - %v\n", rpcCallStr, err.Error()) } else if xreply != nil { *reply = int(xreply.Response) @@ -622,17 +624,17 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { var cinfo GRPCClient opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPortGRPC) + //cinfo.conn, err = grpc.DialContext(context.TODO(), cStr, grpc.WithTransportCredentials(insecure.NewCredentials())) cinfo.conn, err = grpc.Dial(cStr, opts...) if err != nil { tk.LogIt(tk.LogInfo,"Failed to dial xsync gRPC pair: %v", err) - return -1 + continue } cinfo.xclient = NewXSyncClient(cinfo.conn) pe.Client = cinfo tk.LogIt(tk.LogInfo, "XSync gRPC - %s :Connected\n", cStr) } - } reply = 0 @@ -699,34 +701,38 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { } if dp.RPC.RPCType == RPCTypeGoRPC { - resp := <-call.Done - if resp != nil { - err = resp.Error - } - } - select { - case <-time.After(timeout): - tk.LogIt(tk.LogError, "rpc call timeout(%v)\n", timeout) - if pe.Client != nil { - if dp.RPC.RPCType == RPCTypeGoRPC { - pe.Client.(*rpc.Client).Close() - } else { - pe.Client.(GRPCClient).conn.Close() + select { + case <-time.After(timeout): + tk.LogIt(tk.LogError, "rpc call timeout(%v)\n", timeout) + if pe.Client != nil { + if dp.RPC.RPCType == RPCTypeGoRPC { + pe.Client.(*rpc.Client).Close() + } else { + pe.Client.(GRPCClient).conn.Close() + } + } + pe.Client = nil + rpcRetries++ + if rpcRetries < 2 { + goto restartRPC + } + rpcErr = true + case resp := <-call.Done: + if resp != nil && resp.Error != nil { + tk.LogIt(tk.LogError, "rpc call failed(%s)\n", resp.Error) + rpcErr = true } } - pe.Client = nil - rpcRetries++ - if rpcRetries < 2 { - goto restartRPC - } - rpcErr = true - } - if err != nil { - tk.LogIt(tk.LogError, "rpc call failed(%s)\n", err) - rpcErr = true - } - if reply != 0 { - rpcErr = true + } else { + if err != nil || reply != 0 { + tk.LogIt(tk.LogError, "grpc call failed(%s)\n", err) + rpcErr = true + pe.Client = nil + rpcRetries++ + if rpcRetries < 2 { + goto restartRPC + } + } } } diff --git a/loxinet/xsync_server.go b/loxinet/xsync_server.go index ed7d97732..4f50b69e5 100644 --- a/loxinet/xsync_server.go +++ b/loxinet/xsync_server.go @@ -24,7 +24,6 @@ import ( "net" "github.com/golang/protobuf/ptypes/wrappers" - //"github.com/loxilb-io/loxilb/loxinet" tk "github.com/loxilb-io/loxilib" "google.golang.org/grpc" "google.golang.org/protobuf/proto" @@ -55,9 +54,30 @@ func (xs *XSync) DpWorkOnBlockCtAdd(blockCtis []DpCtInfo, ret *int) error { return nil } +// DpWorkOnBlockCtDelete - Add block CT entries from remote +func (xs *XSync) DpWorkOnBlockCtDelete(blockCtis []DpCtInfo, ret *int) error { + if !mh.ready { + return errors.New("Not-Ready") + } + + *ret = 0 + for _, cti := range blockCtis { + + tk.LogIt(tk.LogDebug, "RPC - Block CT Del %s\n", cti.Key()) + r := mh.dp.DpHooks.DpCtDel(&cti) + if r != 0 { + *ret = r + } + } + + return nil +} + // DpWorkOnCtAdd - Add a CT entry from remote goRPC client func (xs *XSync) DpWorkOnCtAdd(cti DpCtInfo, ret *int) error { if !mh.ready { + *ret = -1 + tk.LogIt(tk.LogDebug, "RPC - CT Xsync Not-Ready") return errors.New("Not-Ready") } @@ -70,6 +90,7 @@ func (xs *XSync) DpWorkOnCtAdd(cti DpCtInfo, ret *int) error { if r.RemoteID == int(cti.Sport) { r.RPCState = true *ret = 0 + tk.LogIt(tk.LogDebug, "RPC - CT Xsync Remote-%v Already present\n", cti.Sport) return nil } } @@ -107,7 +128,7 @@ func (xs *XSync) DpWorkOnCtGet(async int, ret *int) error { } // Most likely need to reset reverse rpc channel - mh.dp.DpXsyncRPCReset() + //mh.dp.DpXsyncRPCReset() tk.LogIt(tk.LogDebug, "RPC - CT Get %d\n", async) mh.dp.DpHooks.DpCtGetAsync() @@ -119,6 +140,9 @@ func (xs *XSync) DpWorkOnCtGet(async int, ret *int) error { func (xs *XSync) DpWorkOnBlockCtAddGRPC(ctx context.Context, m *ConnInfo) (*XSyncReply, error) { var value interface{} var resp int + var cti DpCtInfo + var ctis []DpCtInfo + bytesValue := &wrappers.BytesValue{} anyValue := m.Cti @@ -130,14 +154,26 @@ func (xs *XSync) DpWorkOnBlockCtAddGRPC(ctx context.Context, m *ConnInfo) (*XSyn if uErr != nil { return &XSyncReply{Response: -1}, uErr } + //tk.LogIt(tk.LogDebug, "RPC - BLOCK CT ADD : %T : %v\n", value, value) + blk := value.([]interface{}) + for i := range blk { + //tk.LogIt(tk.LogDebug, "BLOCK CT %d: %v\n", i, blk[i]) + jsonData, _:= json.Marshal(blk[i].(map[string]interface{})) + //tk.LogIt(tk.LogDebug, "BLOCK CT json %d: %v\n", i, jsonData) + json.Unmarshal(jsonData, &cti) + ctis = append(ctis, cti) + + } } - xs.DpWorkOnBlockCtAdd(value.([]DpCtInfo), &resp) + + xs.DpWorkOnBlockCtAdd(ctis, &resp) return &XSyncReply{Response: int32(resp)}, nil } func (xs *XSync) DpWorkOnCtAddGRPC(ctx context.Context, m *ConnInfo) (*XSyncReply, error) { var value interface{} var resp int + var cti DpCtInfo bytesValue := &wrappers.BytesValue{} anyValue := m.Cti @@ -149,19 +185,53 @@ func (xs *XSync) DpWorkOnCtAddGRPC(ctx context.Context, m *ConnInfo) (*XSyncRepl if uErr != nil { return &XSyncReply{Response: -1}, uErr } + jsonData, _:= json.Marshal( value.(map[string]interface{})) + json.Unmarshal(jsonData, &cti) } - - xs.DpWorkOnCtAdd(value.(DpCtInfo), &resp) + //tk.LogIt(tk.LogDebug, "RPC - CT ADD : %v\n", cti) + + xs.DpWorkOnCtAdd(cti, &resp) + //tk.LogIt(tk.LogDebug, "RPC - CT ADD Server reply: %v\n", resp) return &XSyncReply{Response: int32(resp)}, nil } -func (xs *XSync) DpWorkOnBlockCtDelGRPC(context.Context, *ConnInfo) (*XSyncReply, error) { - return nil, errors.New("method DpWorkOnBlockCtDelGRPC not implemented") +func (xs *XSync) DpWorkOnBlockCtDelGRPC(ctx context.Context, m *ConnInfo) (*XSyncReply, error) { + var value interface{} + var resp int + var cti DpCtInfo + var ctis []DpCtInfo + + bytesValue := &wrappers.BytesValue{} + anyValue := m.Cti + + err := anypb.UnmarshalTo(anyValue, bytesValue, proto.UnmarshalOptions{}) + if err != nil { + return &XSyncReply{Response: -1}, err + } else { + uErr := json.Unmarshal(bytesValue.Value, &value) + if uErr != nil { + return &XSyncReply{Response: -1}, uErr + } + //tk.LogIt(tk.LogDebug, "RPC - BLOCK CT DEL : %T : %v\n", value, value) + blk := value.([]interface{}) + for i := range blk { + //tk.LogIt(tk.LogDebug, "BLOCK CT %d: %v\n", i, blk[i]) + jsonData, _:= json.Marshal(blk[i].(map[string]interface{})) + //tk.LogIt(tk.LogDebug, "BLOCK CT json %d: %v\n", i, jsonData) + json.Unmarshal(jsonData, &cti) + ctis = append(ctis, cti) + } + } + + xs.DpWorkOnBlockCtDelete(ctis, &resp) + return &XSyncReply{Response: int32(resp)}, nil } func (xs *XSync) DpWorkOnCtDelGRPC(ctx context.Context, m *ConnInfo) (*XSyncReply, error) { var value interface{} var resp int + var cti DpCtInfo + bytesValue := &wrappers.BytesValue{} anyValue := m.Cti @@ -173,8 +243,11 @@ func (xs *XSync) DpWorkOnCtDelGRPC(ctx context.Context, m *ConnInfo) (*XSyncRepl if uErr != nil { return &XSyncReply{Response: -1}, uErr } + jsonData, _:= json.Marshal( value.(map[string]interface{})) + json.Unmarshal(jsonData, &cti) } - xs.DpWorkOnCtDelete(value.(DpCtInfo), &resp) + + xs.DpWorkOnCtDelete(cti, &resp) return &XSyncReply{Response: int32(resp)}, nil } From d565221ad0b5c4f7e37e107f9c8a4b5e4d6f361a Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Fri, 1 Sep 2023 16:27:54 +0900 Subject: [PATCH 5/8] xsync optimizations --- go.mod | 2 +- loxinet/dpbroker.go | 181 +++++++++------- loxinet/dpebpf_linux.go | 13 +- loxinet/loxinet.go | 48 +---- loxinet/xsync.pb.go | 442 ++++++++++++++++++++++++++++++++------- loxinet/xsync.proto | 43 +++- loxinet/xsync_grpc.pb.go | 146 ++++--------- loxinet/xsync_server.go | 206 ++++++++---------- main.go | 2 +- options/options.go | 1 + 10 files changed, 651 insertions(+), 433 deletions(-) diff --git a/go.mod b/go.mod index 86e2ed743..e4b9edd9c 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/go-openapi/analysis v0.21.4 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/golang/protobuf v1.5.2 + github.com/golang/protobuf v1.5.2 // indirect github.com/google/uuid v1.3.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/loxinet/dpbroker.go b/loxinet/dpbroker.go index f38db123d..2eca9b388 100644 --- a/loxinet/dpbroker.go +++ b/loxinet/dpbroker.go @@ -33,11 +33,6 @@ import ( tk "github.com/loxilb-io/loxilib" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "google.golang.org/protobuf/types/known/anypb" - "github.com/golang/protobuf/ptypes/any" - "github.com/golang/protobuf/ptypes/wrappers" - "google.golang.org/protobuf/proto" - "encoding/json" ) // man names constants @@ -87,10 +82,9 @@ const ( // maximum dp work queue lengths const ( - DpWorkQLen = 1024 - XSyncPort = 22222 - XSyncPortGRPC = 22224 - DpTiVal = 20 + DpWorkQLen = 1024 + XSyncPort = 22222 + DpTiVal = 20 ) // MirrDpWorkQ - work queue entry for mirror operation @@ -303,34 +297,34 @@ type NatDpWorkQ struct { // DpCtInfo - representation of a datapath conntrack information type DpCtInfo struct { - DIP net.IP `json:"dip"` - SIP net.IP `json:"sip"` - Dport uint16 `json:"dport"` - Sport uint16 `json:"sport"` - Proto string `json:"proto"` - CState string `json:"cstate"` - CAct string `json:"cact"` - CI string `json:"ci"` - Packets uint64 `json:"packets"` - Bytes uint64 `json:"bytes"` - Deleted int `json:"deleted"` - PKey []byte `json:"pkey"` - PVal []byte `json:"pval"` - LTs time.Time `json:"lts"` - NTs time.Time `json:"nts"` - XSync bool `json:"xsync"` + DIP net.IP `json:"dip"` + SIP net.IP `json:"sip"` + Dport uint16 `json:"dport"` + Sport uint16 `json:"sport"` + Proto string `json:"proto"` + CState string `json:"cstate"` + CAct string `json:"cact"` + CI string `json:"ci"` + Packets uint64 `json:"packets"` + Bytes uint64 `json:"bytes"` + Deleted int `json:"deleted"` + PKey []byte `json:"pkey"` + PVal []byte `json:"pval"` + LTs time.Time `json:"lts"` + NTs time.Time `json:"nts"` + XSync bool `json:"xsync"` // LB Association Data - ServiceIP net.IP `json:"serviceip"` - ServProto string `json:"servproto"` - L4ServPort uint16 `json:"l4servproto"` - BlockNum uint16 `json:"blocknum"` + ServiceIP net.IP `json:"serviceip"` + ServProto string `json:"servproto"` + L4ServPort uint16 `json:"l4servproto"` + BlockNum uint16 `json:"blocknum"` } const ( - RPCTypeGoRPC = iota + RPCTypeNetRPC = iota RPCTypeGRPC -) +) // XSync - Remote sync peer information type XSync struct { @@ -417,13 +411,13 @@ type DpHookInterface interface { } type GRPCClient struct { - conn *grpc.ClientConn + conn *grpc.ClientConn xclient XSyncClient } // DpPeer - Remote DP Peer information type DpPeer struct { - Peer net.IP + Peer net.IP //Client *rpc.Client Client interface{} } @@ -435,6 +429,7 @@ type DpH struct { ToFinCh chan int DpHooks DpHookInterface SyncMtx sync.RWMutex + MapMtx sync.RWMutex Peers []DpPeer RPC *XSync Remotes []XSync @@ -478,9 +473,9 @@ func (dp *DpH) DpXsyncRPCReset() int { defer dp.SyncMtx.Unlock() for idx := range mh.dp.Peers { pe := &mh.dp.Peers[idx] - if dp.RPC.RPCType == RPCTypeGoRPC { + if dp.RPC.RPCType == RPCTypeNetRPC { cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) - client, ok := pe.Client.(*rpc.Client) + client, ok := pe.Client.(*rpc.Client) if ok && client != nil { client.Close() pe.Client = nil @@ -488,7 +483,7 @@ func (dp *DpH) DpXsyncRPCReset() int { if pe.Client == nil { pe.Client, err = dialHTTPPath("tcp", cStr, rpc.DefaultRPCPath) if pe.Client == nil { - tk.LogIt(tk.LogInfo,"Failed to dial xsync goRPC pair: %v", err) + tk.LogIt(tk.LogInfo, "Failed to dial xsync goRPC pair: %v", err) return -1 } tk.LogIt(tk.LogInfo, "XSync goRPC - %s :Reset\n", cStr) @@ -496,21 +491,21 @@ func (dp *DpH) DpXsyncRPCReset() int { } else { var opts []grpc.DialOption var cinfo GRPCClient - client, ok := pe.Client.(GRPCClient) + client, ok := pe.Client.(GRPCClient) if ok { client.conn.Close() pe.Client = nil } - + if pe.Client == nil { - cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPortGRPC) + cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) cinfo.conn, err = grpc.Dial(cStr, opts...) if err != nil { - tk.LogIt(tk.LogInfo,"Failed to dial xsync gRPC pair: %v", err) + tk.LogIt(tk.LogInfo, "Failed to dial xsync gRPC pair: %v", err) return -1 } - + cinfo.xclient = NewXSyncClient(cinfo.conn) pe.Client = cinfo tk.LogIt(tk.LogInfo, "XSync gRPC - %s :Reset\n", cStr) @@ -543,39 +538,69 @@ func (dp *DpH) WaitXsyncReady(who string) { } } +func (ci *DpCtInfo) ConvertToCtInfo(c *CtInfo) { + c.Dip = ci.DIP + c.Sip = ci.SIP + c.Dport = int32(ci.Dport) + c.Sport = int32(ci.Sport) + c.Proto = ci.Proto + c.Cstate = ci.CState + c.Cact = ci.CAct + c.Ci = ci.CI + c.Packets = int64(ci.Packets) + c.Bytes = int64(ci.Bytes) + c.Deleted = int32(ci.Deleted) + c.Pkey = ci.PKey + c.Pval = ci.PVal + c.Xsync = ci.XSync + c.Serviceip = ci.ServiceIP + c.Servproto = ci.ServProto + c.L4Servport = int32(ci.L4ServPort) + c.Blocknum = int32(ci.BlockNum) +} + func callGRPC(client XSyncClient, rpcCallStr string, args interface{}, reply *int) error { var err error var xreply *XSyncReply + var ctis []*CtInfo + var ct *CtInfo + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - anyValue := &any.Any{} - if rpcCallStr != "XSync.DpWorkOnCtGet" { - bytes, _ := json.Marshal(args) - bytesValue := &wrappers.BytesValue{ - Value: bytes, + if (rpcCallStr == "XSync.DpWorkOnBlockCtAdd") || + (rpcCallStr == "XSync.DpWorkOnBlockCtDelete") { + blkCtis := args.([]DpCtInfo) + ctis = make([]*CtInfo, len(blkCtis)) + for i, c := range blkCtis { + ctis[i] = &CtInfo{} + c.ConvertToCtInfo(ctis[i]) } - err = anypb.MarshalFrom(anyValue, bytesValue, proto.MarshalOptions{}) - } + } else if (rpcCallStr == "XSync.DpWorkOnCtAdd") || + (rpcCallStr == "XSync.DpWorkOnCtDelete") { + c := args.(DpCtInfo) + ct = &CtInfo{} + c.ConvertToCtInfo(ct) + } if rpcCallStr == "XSync.DpWorkOnBlockCtAdd" { - xreply, err = client.DpWorkOnBlockCtAddGRPC(ctx, &ConnInfo{Cti: anyValue}) + xreply, err = client.DpWorkOnBlockCtModGRPC(ctx, &BlockCtInfoMod{Add: true, Ct: ctis}) } else if rpcCallStr == "XSync.DpWorkOnBlockCtDelete" { - xreply, err = client.DpWorkOnBlockCtDelGRPC(ctx, &ConnInfo{Cti: anyValue}) + xreply, err = client.DpWorkOnBlockCtModGRPC(ctx, &BlockCtInfoMod{Add: false, Ct: ctis}) } else if rpcCallStr == "XSync.DpWorkOnCtAdd" { - xreply, err = client.DpWorkOnCtAddGRPC(ctx, &ConnInfo{Cti: anyValue}) + xreply, err = client.DpWorkOnCtModGRPC(ctx, &CtInfoMod{Add: true, Ct: ct}) } else if rpcCallStr == "XSync.DpWorkOnCtDelete" { - xreply, err = client.DpWorkOnCtDelGRPC(ctx, &ConnInfo{Cti: anyValue}) + xreply, err = client.DpWorkOnCtModGRPC(ctx, &CtInfoMod{Add: false, Ct: ct}) } else if rpcCallStr == "XSync.DpWorkOnCtGet" { xreply, err = client.DpWorkOnCtGetGRPC(ctx, &ConnGet{Async: args.(int32)}) } - if (err != nil) { + if err != nil { *reply = -1 - tk.LogIt(tk.LogInfo, "XSync %s reply - %v\n", rpcCallStr, err.Error()) + tk.LogIt(tk.LogInfo, "XSync %s reply - %v[NOK]\n", rpcCallStr, err.Error()) } else if xreply != nil { *reply = int(xreply.Response) - tk.LogIt(tk.LogInfo, "XSync %s reply - %v[OK]\n", rpcCallStr, *reply) + tk.LogIt(tk.LogInfo, "XSync %s peer reply - %d\n", rpcCallStr, *reply) } return err } @@ -610,12 +635,13 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { pe := &mh.dp.Peers[idx] if pe.Client == nil { var err error - if dp.RPC.RPCType == RPCTypeGoRPC { + if dp.RPC.RPCType == RPCTypeNetRPC { cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) pe.Client, err = dialHTTPPath("tcp", cStr, rpc.DefaultRPCPath) - if pe.Client == nil { + if pe.Client == nil || err != nil { tk.LogIt(tk.LogInfo, "XSync RPC - %s :Fail(%s)\n", cStr, err) rpcErr = true + pe.Client = nil continue } tk.LogIt(tk.LogInfo, "XSync goRPC - %s :Connected\n", cStr) @@ -623,14 +649,13 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { var opts []grpc.DialOption var cinfo GRPCClient opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) - cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPortGRPC) - //cinfo.conn, err = grpc.DialContext(context.TODO(), cStr, grpc.WithTransportCredentials(insecure.NewCredentials())) + cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) cinfo.conn, err = grpc.Dial(cStr, opts...) if err != nil { - tk.LogIt(tk.LogInfo,"Failed to dial xsync gRPC pair: %v", err) + tk.LogIt(tk.LogInfo, "Failed to dial xsync gRPC pair: %v", err) continue } - + cinfo.xclient = NewXSyncClient(cinfo.conn) pe.Client = cinfo tk.LogIt(tk.LogInfo, "XSync gRPC - %s :Connected\n", cStr) @@ -676,15 +701,15 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { return 0 } } - if dp.RPC.RPCType == RPCTypeGoRPC { - client, _ := pe.Client.(*rpc.Client) + if dp.RPC.RPCType == RPCTypeNetRPC { + client, _ := pe.Client.(*rpc.Client) if cti != nil { call = client.Go(rpcCallStr, *cti, &reply, make(chan *rpc.Call, 1)) } else { call = client.Go(rpcCallStr, blkCti, &reply, make(chan *rpc.Call, 1)) } } else { - client, _ := pe.Client.(GRPCClient) + client, _ := pe.Client.(GRPCClient) if cti != nil { err = callGRPC(client.xclient, rpcCallStr, *cti, &reply) } else { @@ -693,19 +718,19 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { } } else { async := 1 - if dp.RPC.RPCType == RPCTypeGoRPC { + if dp.RPC.RPCType == RPCTypeNetRPC { call = pe.Client.(*rpc.Client).Go(rpcCallStr, async, &reply, make(chan *rpc.Call, 1)) } else { err = callGRPC(pe.Client.(GRPCClient).xclient, rpcCallStr, int32(async), &reply) } } - if dp.RPC.RPCType == RPCTypeGoRPC { + if dp.RPC.RPCType == RPCTypeNetRPC { select { case <-time.After(timeout): tk.LogIt(tk.LogError, "rpc call timeout(%v)\n", timeout) if pe.Client != nil { - if dp.RPC.RPCType == RPCTypeGoRPC { + if dp.RPC.RPCType == RPCTypeNetRPC { pe.Client.(*rpc.Client).Close() } else { pe.Client.(GRPCClient).conn.Close() @@ -723,8 +748,8 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { rpcErr = true } } - } else { - if err != nil || reply != 0 { + } else { + if err != nil { tk.LogIt(tk.LogError, "grpc call failed(%s)\n", err) rpcErr = true pe.Client = nil @@ -732,7 +757,11 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { if rpcRetries < 2 { goto restartRPC } - } + } + if reply != 0 { + tk.LogIt(tk.LogError, "grpc server returned error (%d)\n", reply) + rpcErr = true + } } } @@ -743,7 +772,7 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { } // DpBrokerInit - initialize the DP broker subsystem -func DpBrokerInit(dph DpHookInterface) *DpH { +func DpBrokerInit(dph DpHookInterface, rpcMode int) *DpH { nDp := new(DpH) nDp.ToDpCh = make(chan interface{}, DpWorkQLen) @@ -751,18 +780,14 @@ func DpBrokerInit(dph DpHookInterface) *DpH { nDp.ToFinCh = make(chan int) nDp.DpHooks = dph nDp.RPC = new(XSync) - - nDp.RPC.RPCType = RPCTypeGRPC - if nDp.RPC.RPCType == RPCTypeGRPC { - go startxSyncGRPCServer() - } + + nDp.RPC.RPCType = rpcMode go DpWorker(nDp, nDp.ToFinCh, nDp.ToDpCh) return nDp } - // DpWorkOnPort - routine to work on a port work queue request func (dp *DpH) DpWorkOnPort(pWq *PortDpWorkQ) DpRetT { if pWq.Work == DpCreate { @@ -901,7 +926,7 @@ func (dp *DpH) DpWorkOnPeerOp(pWq *PeerDpWorkQ) DpRetT { pe := &dp.Peers[idx] if pe.Peer.Equal(pWq.PeerIP) { if pe.Client != nil { - if dp.RPC.RPCType == RPCTypeGoRPC { + if dp.RPC.RPCType == RPCTypeNetRPC { pe.Client.(*rpc.Client).Close() } else { pe.Client.(GRPCClient).conn.Close() diff --git a/loxinet/dpebpf_linux.go b/loxinet/dpebpf_linux.go index ef521166b..79f9fc7a0 100644 --- a/loxinet/dpebpf_linux.go +++ b/loxinet/dpebpf_linux.go @@ -92,7 +92,7 @@ const ( const ( DpEbpfLinuxTiVal = 10 ctiDeleteSyncRetries = 3 - blkCtiMaxLen = 4096 + blkCtiMaxLen = 8192 mapNotifierChLen = 8096 mapNotifierWorkers = 1 ) @@ -187,9 +187,11 @@ func dpEbpfTicker() { // Age any entries related to Conntrack /* No need to fetch all stats in this fashion */ //C.llb_collect_map_stats(C.int(C.LL_DP_CT_STATS_MAP)) - + mh.dp.MapMtx.Lock() /* Per entry stats will be fetched in C.ll_ct_map_ent_has_aged */ C.llb_age_map_entries(C.LL_DP_CT_MAP) + mh.dp.MapMtx.Unlock() + C.llb_age_map_entries(C.LL_DP_FCV4_MAP) // This means around 10s from start @@ -1829,7 +1831,7 @@ func dpCTMapChkUpdates() { } } - if len(blkCti) > blkCtiMaxLen { + if len(blkCti) >= blkCtiMaxLen { tk.LogIt(tk.LogDebug, "[CT] Block Add Sync - \n") tc1 := time.Now() mh.dp.DpXsyncRPC(DpSyncAdd, blkCti) @@ -1838,7 +1840,7 @@ func dpCTMapChkUpdates() { blkCti = nil } - if len(blkDelCti) > blkCtiMaxLen { + if len(blkDelCti) >= blkCtiMaxLen { tk.LogIt(tk.LogDebug, "[CT] Block Del Sync - \n") mh.dp.DpXsyncRPC(DpSyncDelete, blkDelCti) blkDelCti = nil @@ -1927,7 +1929,8 @@ func (e *DpEbpfH) DpCtAdd(w *DpCtInfo) int { cte.XSync = false cte.NTs = time.Now() - cte.LTs = cti.NTs + //cte.LTs = cti.NTs + cte.LTs = time.Now() ret := C.llb_add_map_elem(C.LL_DP_CT_MAP, unsafe.Pointer(&cti.PKey[0]), unsafe.Pointer(&cti.PVal[0])) if ret != 0 { diff --git a/loxinet/loxinet.go b/loxinet/loxinet.go index f17963d8a..0aeb187a9 100644 --- a/loxinet/loxinet.go +++ b/loxinet/loxinet.go @@ -25,11 +25,8 @@ import ( cmn "github.com/loxilb-io/loxilb/common" opts "github.com/loxilb-io/loxilb/options" tk "github.com/loxilb-io/loxilib" - "io" "net" - "net/http" _ "net/http/pprof" - "net/rpc" "os" "os/signal" "runtime/debug" @@ -80,42 +77,6 @@ type loxiNetH struct { pFile *os.File } -// LoxiXsyncMain - State Sync subsystem init -func LoxiXsyncMain() { - if opts.Opts.ClusterNodes == "none" { - return - } - - // Stack trace logger - defer func() { - if e := recover(); e != nil { - if mh.logger != nil { - tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack()) - } - } - }() - - for { - rpcObj := new(XSync) - err := rpc.Register(rpcObj) - if err != nil { - panic("Failed to register rpc") - } - - rpc.HandleHTTP() - - http.HandleFunc("/", func(res http.ResponseWriter, req *http.Request) { - io.WriteString(res, "loxilb-xsync\n") - }) - - listener := fmt.Sprintf(":%d", XSyncPort) - err = http.ListenAndServe(listener, nil) - if err != nil { - panic("Failed to rpc-listen") - } - } -} - // NodeWalker - an implementation of node walker interface func (mh *loxiNetH) NodeWalker(b string) { tk.LogIt(tk.LogDebug, "%s\n", b) @@ -203,6 +164,8 @@ func loxiNetTicker(bgpPeerMode bool) { var mh loxiNetH func loxiNetInit() { + var rpcMode int + spawnKa, kaMode := KAString2Mode(opts.Opts.Ka) clusterMode := false if opts.Opts.ClusterNodes != "none" { @@ -251,11 +214,16 @@ func loxiNetInit() { return } } + if opts.Opts.Rpc == "netrpc" { + rpcMode = RPCTypeNetRPC + } else { + rpcMode = RPCTypeGRPC + } if !opts.Opts.BgpPeerMode { // Initialize the ebpf datapath subsystem mh.dpEbpf = DpEbpfInit(clusterMode, mh.self, mh.rssEn, mh.eHooks, -1) - mh.dp = DpBrokerInit(mh.dpEbpf) + mh.dp = DpBrokerInit(mh.dpEbpf, rpcMode) // Initialize the security zone subsystem mh.zn = ZoneInit() diff --git a/loxinet/xsync.pb.go b/loxinet/xsync.pb.go index cfd336d76..5e0e20825 100644 --- a/loxinet/xsync.pb.go +++ b/loxinet/xsync.pb.go @@ -7,7 +7,6 @@ package loxinet import ( - any1 "github.com/golang/protobuf/ptypes/any" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -68,16 +67,16 @@ func (x *XSyncReply) GetResponse() int32 { return 0 } -type ConnInfo struct { +type ConnGet struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Cti *any1.Any `protobuf:"bytes,1,opt,name=cti,proto3" json:"cti,omitempty"` + Async int32 `protobuf:"varint,1,opt,name=async,proto3" json:"async,omitempty"` } -func (x *ConnInfo) Reset() { - *x = ConnInfo{} +func (x *ConnGet) Reset() { + *x = ConnGet{} if protoimpl.UnsafeEnabled { mi := &file_loxinet_xsync_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -85,13 +84,13 @@ func (x *ConnInfo) Reset() { } } -func (x *ConnInfo) String() string { +func (x *ConnGet) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ConnInfo) ProtoMessage() {} +func (*ConnGet) ProtoMessage() {} -func (x *ConnInfo) ProtoReflect() protoreflect.Message { +func (x *ConnGet) ProtoReflect() protoreflect.Message { mi := &file_loxinet_xsync_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -103,28 +102,45 @@ func (x *ConnInfo) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ConnInfo.ProtoReflect.Descriptor instead. -func (*ConnInfo) Descriptor() ([]byte, []int) { +// Deprecated: Use ConnGet.ProtoReflect.Descriptor instead. +func (*ConnGet) Descriptor() ([]byte, []int) { return file_loxinet_xsync_proto_rawDescGZIP(), []int{1} } -func (x *ConnInfo) GetCti() *any1.Any { +func (x *ConnGet) GetAsync() int32 { if x != nil { - return x.Cti + return x.Async } - return nil + return 0 } -type ConnGet struct { +type CtInfo struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Async int32 `protobuf:"varint,1,opt,name=async,proto3" json:"async,omitempty"` + Dip []byte `protobuf:"bytes,1,opt,name=dip,proto3" json:"dip,omitempty"` + Sip []byte `protobuf:"bytes,2,opt,name=sip,proto3" json:"sip,omitempty"` + Dport int32 `protobuf:"varint,3,opt,name=dport,proto3" json:"dport,omitempty"` + Sport int32 `protobuf:"varint,4,opt,name=sport,proto3" json:"sport,omitempty"` + Proto string `protobuf:"bytes,5,opt,name=proto,proto3" json:"proto,omitempty"` + Cstate string `protobuf:"bytes,6,opt,name=cstate,proto3" json:"cstate,omitempty"` + Cact string `protobuf:"bytes,7,opt,name=cact,proto3" json:"cact,omitempty"` + Ci string `protobuf:"bytes,8,opt,name=ci,proto3" json:"ci,omitempty"` + Packets int64 `protobuf:"varint,9,opt,name=packets,proto3" json:"packets,omitempty"` + Bytes int64 `protobuf:"varint,10,opt,name=bytes,proto3" json:"bytes,omitempty"` + Deleted int32 `protobuf:"varint,11,opt,name=deleted,proto3" json:"deleted,omitempty"` + Pkey []byte `protobuf:"bytes,12,opt,name=pkey,proto3" json:"pkey,omitempty"` + Pval []byte `protobuf:"bytes,13,opt,name=pval,proto3" json:"pval,omitempty"` + Xsync bool `protobuf:"varint,14,opt,name=xsync,proto3" json:"xsync,omitempty"` + Serviceip []byte `protobuf:"bytes,15,opt,name=serviceip,proto3" json:"serviceip,omitempty"` + Servproto string `protobuf:"bytes,16,opt,name=servproto,proto3" json:"servproto,omitempty"` + L4Servport int32 `protobuf:"varint,17,opt,name=l4servport,proto3" json:"l4servport,omitempty"` + Blocknum int32 `protobuf:"varint,18,opt,name=blocknum,proto3" json:"blocknum,omitempty"` } -func (x *ConnGet) Reset() { - *x = ConnGet{} +func (x *CtInfo) Reset() { + *x = CtInfo{} if protoimpl.UnsafeEnabled { mi := &file_loxinet_xsync_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -132,13 +148,13 @@ func (x *ConnGet) Reset() { } } -func (x *ConnGet) String() string { +func (x *CtInfo) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ConnGet) ProtoMessage() {} +func (*CtInfo) ProtoMessage() {} -func (x *ConnGet) ProtoReflect() protoreflect.Message { +func (x *CtInfo) ProtoReflect() protoreflect.Message { mi := &file_loxinet_xsync_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -150,50 +166,302 @@ func (x *ConnGet) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ConnGet.ProtoReflect.Descriptor instead. -func (*ConnGet) Descriptor() ([]byte, []int) { +// Deprecated: Use CtInfo.ProtoReflect.Descriptor instead. +func (*CtInfo) Descriptor() ([]byte, []int) { return file_loxinet_xsync_proto_rawDescGZIP(), []int{2} } -func (x *ConnGet) GetAsync() int32 { +func (x *CtInfo) GetDip() []byte { if x != nil { - return x.Async + return x.Dip + } + return nil +} + +func (x *CtInfo) GetSip() []byte { + if x != nil { + return x.Sip + } + return nil +} + +func (x *CtInfo) GetDport() int32 { + if x != nil { + return x.Dport + } + return 0 +} + +func (x *CtInfo) GetSport() int32 { + if x != nil { + return x.Sport + } + return 0 +} + +func (x *CtInfo) GetProto() string { + if x != nil { + return x.Proto + } + return "" +} + +func (x *CtInfo) GetCstate() string { + if x != nil { + return x.Cstate + } + return "" +} + +func (x *CtInfo) GetCact() string { + if x != nil { + return x.Cact + } + return "" +} + +func (x *CtInfo) GetCi() string { + if x != nil { + return x.Ci + } + return "" +} + +func (x *CtInfo) GetPackets() int64 { + if x != nil { + return x.Packets + } + return 0 +} + +func (x *CtInfo) GetBytes() int64 { + if x != nil { + return x.Bytes + } + return 0 +} + +func (x *CtInfo) GetDeleted() int32 { + if x != nil { + return x.Deleted + } + return 0 +} + +func (x *CtInfo) GetPkey() []byte { + if x != nil { + return x.Pkey + } + return nil +} + +func (x *CtInfo) GetPval() []byte { + if x != nil { + return x.Pval + } + return nil +} + +func (x *CtInfo) GetXsync() bool { + if x != nil { + return x.Xsync + } + return false +} + +func (x *CtInfo) GetServiceip() []byte { + if x != nil { + return x.Serviceip + } + return nil +} + +func (x *CtInfo) GetServproto() string { + if x != nil { + return x.Servproto + } + return "" +} + +func (x *CtInfo) GetL4Servport() int32 { + if x != nil { + return x.L4Servport } return 0 } +func (x *CtInfo) GetBlocknum() int32 { + if x != nil { + return x.Blocknum + } + return 0 +} + +type CtInfoMod struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Add bool `protobuf:"varint,1,opt,name=add,proto3" json:"add,omitempty"` + Ct *CtInfo `protobuf:"bytes,2,opt,name=ct,proto3" json:"ct,omitempty"` +} + +func (x *CtInfoMod) Reset() { + *x = CtInfoMod{} + if protoimpl.UnsafeEnabled { + mi := &file_loxinet_xsync_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CtInfoMod) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CtInfoMod) ProtoMessage() {} + +func (x *CtInfoMod) ProtoReflect() protoreflect.Message { + mi := &file_loxinet_xsync_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CtInfoMod.ProtoReflect.Descriptor instead. +func (*CtInfoMod) Descriptor() ([]byte, []int) { + return file_loxinet_xsync_proto_rawDescGZIP(), []int{3} +} + +func (x *CtInfoMod) GetAdd() bool { + if x != nil { + return x.Add + } + return false +} + +func (x *CtInfoMod) GetCt() *CtInfo { + if x != nil { + return x.Ct + } + return nil +} + +type BlockCtInfoMod struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Add bool `protobuf:"varint,1,opt,name=add,proto3" json:"add,omitempty"` + Ct []*CtInfo `protobuf:"bytes,2,rep,name=ct,proto3" json:"ct,omitempty"` +} + +func (x *BlockCtInfoMod) Reset() { + *x = BlockCtInfoMod{} + if protoimpl.UnsafeEnabled { + mi := &file_loxinet_xsync_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BlockCtInfoMod) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BlockCtInfoMod) ProtoMessage() {} + +func (x *BlockCtInfoMod) ProtoReflect() protoreflect.Message { + mi := &file_loxinet_xsync_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BlockCtInfoMod.ProtoReflect.Descriptor instead. +func (*BlockCtInfoMod) Descriptor() ([]byte, []int) { + return file_loxinet_xsync_proto_rawDescGZIP(), []int{4} +} + +func (x *BlockCtInfoMod) GetAdd() bool { + if x != nil { + return x.Add + } + return false +} + +func (x *BlockCtInfoMod) GetCt() []*CtInfo { + if x != nil { + return x.Ct + } + return nil +} + var File_loxinet_xsync_proto protoreflect.FileDescriptor var file_loxinet_xsync_proto_rawDesc = []byte{ 0x0a, 0x13, 0x6c, 0x6f, 0x78, 0x69, 0x6e, 0x65, 0x74, 0x2f, 0x78, 0x73, 0x79, 0x6e, 0x63, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x22, 0x28, 0x0a, 0x0a, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x1a, - 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x32, 0x0a, 0x08, 0x43, 0x6f, - 0x6e, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x26, 0x0a, 0x03, 0x63, 0x74, 0x69, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x03, 0x63, 0x74, 0x69, 0x22, 0x1f, - 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x47, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x61, 0x73, 0x79, - 0x6e, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x32, - 0xfb, 0x01, 0x0a, 0x05, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x32, 0x0a, 0x16, 0x44, 0x70, 0x57, - 0x6f, 0x72, 0x6b, 0x4f, 0x6e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x43, 0x74, 0x41, 0x64, 0x64, 0x47, - 0x52, 0x50, 0x43, 0x12, 0x09, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x0b, - 0x2e, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x2d, 0x0a, - 0x11, 0x44, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x4f, 0x6e, 0x43, 0x74, 0x41, 0x64, 0x64, 0x47, 0x52, - 0x50, 0x43, 0x12, 0x09, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x0b, 0x2e, - 0x58, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x32, 0x0a, 0x16, - 0x44, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x4f, 0x6e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x43, 0x74, 0x44, - 0x65, 0x6c, 0x47, 0x52, 0x50, 0x43, 0x12, 0x09, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x49, 0x6e, 0x66, - 0x6f, 0x1a, 0x0b, 0x2e, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, - 0x12, 0x2d, 0x0a, 0x11, 0x44, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x4f, 0x6e, 0x43, 0x74, 0x44, 0x65, - 0x6c, 0x47, 0x52, 0x50, 0x43, 0x12, 0x09, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x49, 0x6e, 0x66, 0x6f, - 0x1a, 0x0b, 0x2e, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, - 0x2c, 0x0a, 0x11, 0x44, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x4f, 0x6e, 0x43, 0x74, 0x47, 0x65, 0x74, - 0x47, 0x52, 0x50, 0x43, 0x12, 0x08, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x47, 0x65, 0x74, 0x1a, 0x0b, - 0x2e, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0b, 0x5a, - 0x09, 0x2e, 0x2f, 0x6c, 0x6f, 0x78, 0x69, 0x6e, 0x65, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x28, 0x0a, 0x0a, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, + 0x70, 0x6c, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x1f, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x47, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x61, 0x73, + 0x79, 0x6e, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x61, 0x73, 0x79, 0x6e, 0x63, + 0x22, 0xaa, 0x03, 0x0a, 0x06, 0x43, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x10, 0x0a, 0x03, 0x64, + 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x64, 0x69, 0x70, 0x12, 0x10, 0x0a, + 0x03, 0x73, 0x69, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x73, 0x69, 0x70, 0x12, + 0x14, 0x0a, 0x05, 0x64, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, + 0x64, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x63, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x61, 0x63, + 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x61, 0x63, 0x74, 0x12, 0x0e, 0x0a, + 0x02, 0x63, 0x69, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x63, 0x69, 0x12, 0x18, 0x0a, + 0x07, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, + 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, 0x12, 0x18, 0x0a, + 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, + 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6b, 0x65, 0x79, 0x18, + 0x0c, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x70, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, + 0x76, 0x61, 0x6c, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x70, 0x76, 0x61, 0x6c, 0x12, + 0x14, 0x0a, 0x05, 0x78, 0x73, 0x79, 0x6e, 0x63, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, + 0x78, 0x73, 0x79, 0x6e, 0x63, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x69, 0x70, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x69, 0x70, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x1e, 0x0a, 0x0a, 0x6c, 0x34, 0x73, 0x65, 0x72, 0x76, 0x70, 0x6f, 0x72, 0x74, 0x18, + 0x11, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6c, 0x34, 0x73, 0x65, 0x72, 0x76, 0x70, 0x6f, 0x72, + 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0x18, 0x12, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x08, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0x22, 0x36, 0x0a, + 0x09, 0x43, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x4d, 0x6f, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x64, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x61, 0x64, 0x64, 0x12, 0x17, 0x0a, 0x02, + 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x07, 0x2e, 0x43, 0x74, 0x49, 0x6e, 0x66, + 0x6f, 0x52, 0x02, 0x63, 0x74, 0x22, 0x3b, 0x0a, 0x0e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x43, 0x74, + 0x49, 0x6e, 0x66, 0x6f, 0x4d, 0x6f, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x64, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x61, 0x64, 0x64, 0x12, 0x17, 0x0a, 0x02, 0x63, 0x74, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x07, 0x2e, 0x43, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x02, + 0x63, 0x74, 0x32, 0x9f, 0x01, 0x0a, 0x05, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x2c, 0x0a, 0x11, + 0x44, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x4f, 0x6e, 0x43, 0x74, 0x47, 0x65, 0x74, 0x47, 0x52, 0x50, + 0x43, 0x12, 0x08, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x47, 0x65, 0x74, 0x1a, 0x0b, 0x2e, 0x58, 0x53, + 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x2e, 0x0a, 0x11, 0x44, 0x70, + 0x57, 0x6f, 0x72, 0x6b, 0x4f, 0x6e, 0x43, 0x74, 0x4d, 0x6f, 0x64, 0x47, 0x52, 0x50, 0x43, 0x12, + 0x0a, 0x2e, 0x43, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x4d, 0x6f, 0x64, 0x1a, 0x0b, 0x2e, 0x58, 0x53, + 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x38, 0x0a, 0x16, 0x44, 0x70, + 0x57, 0x6f, 0x72, 0x6b, 0x4f, 0x6e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x43, 0x74, 0x4d, 0x6f, 0x64, + 0x47, 0x52, 0x50, 0x43, 0x12, 0x0f, 0x2e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x43, 0x74, 0x49, 0x6e, + 0x66, 0x6f, 0x4d, 0x6f, 0x64, 0x1a, 0x0b, 0x2e, 0x58, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, + 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0b, 0x5a, 0x09, 0x2e, 0x2f, 0x6c, 0x6f, 0x78, 0x69, 0x6e, 0x65, + 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -208,30 +476,28 @@ func file_loxinet_xsync_proto_rawDescGZIP() []byte { return file_loxinet_xsync_proto_rawDescData } -var file_loxinet_xsync_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_loxinet_xsync_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_loxinet_xsync_proto_goTypes = []interface{}{ - (*XSyncReply)(nil), // 0: XSyncReply - (*ConnInfo)(nil), // 1: ConnInfo - (*ConnGet)(nil), // 2: ConnGet - (*any1.Any)(nil), // 3: google.protobuf.Any + (*XSyncReply)(nil), // 0: XSyncReply + (*ConnGet)(nil), // 1: ConnGet + (*CtInfo)(nil), // 2: CtInfo + (*CtInfoMod)(nil), // 3: CtInfoMod + (*BlockCtInfoMod)(nil), // 4: BlockCtInfoMod } var file_loxinet_xsync_proto_depIdxs = []int32{ - 3, // 0: ConnInfo.cti:type_name -> google.protobuf.Any - 1, // 1: XSync.DpWorkOnBlockCtAddGRPC:input_type -> ConnInfo - 1, // 2: XSync.DpWorkOnCtAddGRPC:input_type -> ConnInfo - 1, // 3: XSync.DpWorkOnBlockCtDelGRPC:input_type -> ConnInfo - 1, // 4: XSync.DpWorkOnCtDelGRPC:input_type -> ConnInfo - 2, // 5: XSync.DpWorkOnCtGetGRPC:input_type -> ConnGet - 0, // 6: XSync.DpWorkOnBlockCtAddGRPC:output_type -> XSyncReply - 0, // 7: XSync.DpWorkOnCtAddGRPC:output_type -> XSyncReply - 0, // 8: XSync.DpWorkOnBlockCtDelGRPC:output_type -> XSyncReply - 0, // 9: XSync.DpWorkOnCtDelGRPC:output_type -> XSyncReply - 0, // 10: XSync.DpWorkOnCtGetGRPC:output_type -> XSyncReply - 6, // [6:11] is the sub-list for method output_type - 1, // [1:6] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 2, // 0: CtInfoMod.ct:type_name -> CtInfo + 2, // 1: BlockCtInfoMod.ct:type_name -> CtInfo + 1, // 2: XSync.DpWorkOnCtGetGRPC:input_type -> ConnGet + 3, // 3: XSync.DpWorkOnCtModGRPC:input_type -> CtInfoMod + 4, // 4: XSync.DpWorkOnBlockCtModGRPC:input_type -> BlockCtInfoMod + 0, // 5: XSync.DpWorkOnCtGetGRPC:output_type -> XSyncReply + 0, // 6: XSync.DpWorkOnCtModGRPC:output_type -> XSyncReply + 0, // 7: XSync.DpWorkOnBlockCtModGRPC:output_type -> XSyncReply + 5, // [5:8] is the sub-list for method output_type + 2, // [2:5] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_loxinet_xsync_proto_init() } @@ -253,7 +519,7 @@ func file_loxinet_xsync_proto_init() { } } file_loxinet_xsync_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ConnInfo); i { + switch v := v.(*ConnGet); i { case 0: return &v.state case 1: @@ -265,7 +531,31 @@ func file_loxinet_xsync_proto_init() { } } file_loxinet_xsync_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ConnGet); i { + switch v := v.(*CtInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_loxinet_xsync_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CtInfoMod); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_loxinet_xsync_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BlockCtInfoMod); i { case 0: return &v.state case 1: @@ -283,7 +573,7 @@ func file_loxinet_xsync_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_loxinet_xsync_proto_rawDesc, NumEnums: 0, - NumMessages: 3, + NumMessages: 5, NumExtensions: 0, NumServices: 1, }, diff --git a/loxinet/xsync.proto b/loxinet/xsync.proto index 8647ee8e4..a92f06e48 100644 --- a/loxinet/xsync.proto +++ b/loxinet/xsync.proto @@ -1,24 +1,49 @@ syntax = "proto3"; -import "google/protobuf/any.proto"; option go_package = "./loxinet"; message XSyncReply { int32 response = 1; } -message ConnInfo { - google.protobuf.Any cti = 1; -} - message ConnGet { int32 async = 1; } + +message CtInfo { + bytes dip = 1; + bytes sip = 2; + int32 dport = 3; + int32 sport = 4; + string proto = 5; + string cstate = 6; + string cact = 7; + string ci = 8; + int64 packets = 9; + int64 bytes = 10; + int32 deleted = 11; + bytes pkey = 12; + bytes pval = 13; + bool xsync = 14; + bytes serviceip = 15; + string servproto = 16; + int32 l4servport = 17; + int32 blocknum = 18; +} + +message CtInfoMod { + bool add = 1; + CtInfo ct = 2; +} + +message BlockCtInfoMod { + bool add = 1; + repeated CtInfo ct = 2; +} + // The xsync service definition. service XSync { - rpc DpWorkOnBlockCtAddGRPC (ConnInfo) returns (XSyncReply) {} - rpc DpWorkOnCtAddGRPC (ConnInfo) returns (XSyncReply) {} - rpc DpWorkOnBlockCtDelGRPC (ConnInfo) returns (XSyncReply) {} - rpc DpWorkOnCtDelGRPC (ConnInfo) returns (XSyncReply) {} rpc DpWorkOnCtGetGRPC (ConnGet) returns (XSyncReply) {} + rpc DpWorkOnCtModGRPC (CtInfoMod) returns (XSyncReply) {} + rpc DpWorkOnBlockCtModGRPC (BlockCtInfoMod) returns (XSyncReply) {} } \ No newline at end of file diff --git a/loxinet/xsync_grpc.pb.go b/loxinet/xsync_grpc.pb.go index a83e21601..84c011e2b 100644 --- a/loxinet/xsync_grpc.pb.go +++ b/loxinet/xsync_grpc.pb.go @@ -22,11 +22,9 @@ const _ = grpc.SupportPackageIsVersion7 // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type XSyncClient interface { - DpWorkOnBlockCtAddGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) - DpWorkOnCtAddGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) - DpWorkOnBlockCtDelGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) - DpWorkOnCtDelGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) DpWorkOnCtGetGRPC(ctx context.Context, in *ConnGet, opts ...grpc.CallOption) (*XSyncReply, error) + DpWorkOnCtModGRPC(ctx context.Context, in *CtInfoMod, opts ...grpc.CallOption) (*XSyncReply, error) + DpWorkOnBlockCtModGRPC(ctx context.Context, in *BlockCtInfoMod, opts ...grpc.CallOption) (*XSyncReply, error) } type xSyncClient struct { @@ -37,45 +35,27 @@ func NewXSyncClient(cc grpc.ClientConnInterface) XSyncClient { return &xSyncClient{cc} } -func (c *xSyncClient) DpWorkOnBlockCtAddGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) { - out := new(XSyncReply) - err := c.cc.Invoke(ctx, "/XSync/DpWorkOnBlockCtAddGRPC", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *xSyncClient) DpWorkOnCtAddGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) { - out := new(XSyncReply) - err := c.cc.Invoke(ctx, "/XSync/DpWorkOnCtAddGRPC", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *xSyncClient) DpWorkOnBlockCtDelGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) { +func (c *xSyncClient) DpWorkOnCtGetGRPC(ctx context.Context, in *ConnGet, opts ...grpc.CallOption) (*XSyncReply, error) { out := new(XSyncReply) - err := c.cc.Invoke(ctx, "/XSync/DpWorkOnBlockCtDelGRPC", in, out, opts...) + err := c.cc.Invoke(ctx, "/XSync/DpWorkOnCtGetGRPC", in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *xSyncClient) DpWorkOnCtDelGRPC(ctx context.Context, in *ConnInfo, opts ...grpc.CallOption) (*XSyncReply, error) { +func (c *xSyncClient) DpWorkOnCtModGRPC(ctx context.Context, in *CtInfoMod, opts ...grpc.CallOption) (*XSyncReply, error) { out := new(XSyncReply) - err := c.cc.Invoke(ctx, "/XSync/DpWorkOnCtDelGRPC", in, out, opts...) + err := c.cc.Invoke(ctx, "/XSync/DpWorkOnCtModGRPC", in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *xSyncClient) DpWorkOnCtGetGRPC(ctx context.Context, in *ConnGet, opts ...grpc.CallOption) (*XSyncReply, error) { +func (c *xSyncClient) DpWorkOnBlockCtModGRPC(ctx context.Context, in *BlockCtInfoMod, opts ...grpc.CallOption) (*XSyncReply, error) { out := new(XSyncReply) - err := c.cc.Invoke(ctx, "/XSync/DpWorkOnCtGetGRPC", in, out, opts...) + err := c.cc.Invoke(ctx, "/XSync/DpWorkOnBlockCtModGRPC", in, out, opts...) if err != nil { return nil, err } @@ -86,11 +66,9 @@ func (c *xSyncClient) DpWorkOnCtGetGRPC(ctx context.Context, in *ConnGet, opts . // All implementations must embed UnimplementedXSyncServer // for forward compatibility type XSyncServer interface { - DpWorkOnBlockCtAddGRPC(context.Context, *ConnInfo) (*XSyncReply, error) - DpWorkOnCtAddGRPC(context.Context, *ConnInfo) (*XSyncReply, error) - DpWorkOnBlockCtDelGRPC(context.Context, *ConnInfo) (*XSyncReply, error) - DpWorkOnCtDelGRPC(context.Context, *ConnInfo) (*XSyncReply, error) DpWorkOnCtGetGRPC(context.Context, *ConnGet) (*XSyncReply, error) + DpWorkOnCtModGRPC(context.Context, *CtInfoMod) (*XSyncReply, error) + DpWorkOnBlockCtModGRPC(context.Context, *BlockCtInfoMod) (*XSyncReply, error) mustEmbedUnimplementedXSyncServer() } @@ -98,21 +76,15 @@ type XSyncServer interface { type UnimplementedXSyncServer struct { } -func (UnimplementedXSyncServer) DpWorkOnBlockCtAddGRPC(context.Context, *ConnInfo) (*XSyncReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method DpWorkOnBlockCtAddGRPC not implemented") -} -func (UnimplementedXSyncServer) DpWorkOnCtAddGRPC(context.Context, *ConnInfo) (*XSyncReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method DpWorkOnCtAddGRPC not implemented") -} -func (UnimplementedXSyncServer) DpWorkOnBlockCtDelGRPC(context.Context, *ConnInfo) (*XSyncReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method DpWorkOnBlockCtDelGRPC not implemented") -} -func (UnimplementedXSyncServer) DpWorkOnCtDelGRPC(context.Context, *ConnInfo) (*XSyncReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method DpWorkOnCtDelGRPC not implemented") -} func (UnimplementedXSyncServer) DpWorkOnCtGetGRPC(context.Context, *ConnGet) (*XSyncReply, error) { return nil, status.Errorf(codes.Unimplemented, "method DpWorkOnCtGetGRPC not implemented") } +func (UnimplementedXSyncServer) DpWorkOnCtModGRPC(context.Context, *CtInfoMod) (*XSyncReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method DpWorkOnCtModGRPC not implemented") +} +func (UnimplementedXSyncServer) DpWorkOnBlockCtModGRPC(context.Context, *BlockCtInfoMod) (*XSyncReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method DpWorkOnBlockCtModGRPC not implemented") +} func (UnimplementedXSyncServer) mustEmbedUnimplementedXSyncServer() {} // UnsafeXSyncServer may be embedded to opt out of forward compatibility for this service. @@ -126,92 +98,56 @@ func RegisterXSyncServer(s grpc.ServiceRegistrar, srv XSyncServer) { s.RegisterService(&XSync_ServiceDesc, srv) } -func _XSync_DpWorkOnBlockCtAddGRPC_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ConnInfo) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(XSyncServer).DpWorkOnBlockCtAddGRPC(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/XSync/DpWorkOnBlockCtAddGRPC", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(XSyncServer).DpWorkOnBlockCtAddGRPC(ctx, req.(*ConnInfo)) - } - return interceptor(ctx, in, info, handler) -} - -func _XSync_DpWorkOnCtAddGRPC_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ConnInfo) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(XSyncServer).DpWorkOnCtAddGRPC(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/XSync/DpWorkOnCtAddGRPC", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(XSyncServer).DpWorkOnCtAddGRPC(ctx, req.(*ConnInfo)) - } - return interceptor(ctx, in, info, handler) -} - -func _XSync_DpWorkOnBlockCtDelGRPC_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ConnInfo) +func _XSync_DpWorkOnCtGetGRPC_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ConnGet) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(XSyncServer).DpWorkOnBlockCtDelGRPC(ctx, in) + return srv.(XSyncServer).DpWorkOnCtGetGRPC(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/XSync/DpWorkOnBlockCtDelGRPC", + FullMethod: "/XSync/DpWorkOnCtGetGRPC", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(XSyncServer).DpWorkOnBlockCtDelGRPC(ctx, req.(*ConnInfo)) + return srv.(XSyncServer).DpWorkOnCtGetGRPC(ctx, req.(*ConnGet)) } return interceptor(ctx, in, info, handler) } -func _XSync_DpWorkOnCtDelGRPC_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ConnInfo) +func _XSync_DpWorkOnCtModGRPC_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CtInfoMod) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(XSyncServer).DpWorkOnCtDelGRPC(ctx, in) + return srv.(XSyncServer).DpWorkOnCtModGRPC(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/XSync/DpWorkOnCtDelGRPC", + FullMethod: "/XSync/DpWorkOnCtModGRPC", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(XSyncServer).DpWorkOnCtDelGRPC(ctx, req.(*ConnInfo)) + return srv.(XSyncServer).DpWorkOnCtModGRPC(ctx, req.(*CtInfoMod)) } return interceptor(ctx, in, info, handler) } -func _XSync_DpWorkOnCtGetGRPC_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ConnGet) +func _XSync_DpWorkOnBlockCtModGRPC_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(BlockCtInfoMod) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(XSyncServer).DpWorkOnCtGetGRPC(ctx, in) + return srv.(XSyncServer).DpWorkOnBlockCtModGRPC(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/XSync/DpWorkOnCtGetGRPC", + FullMethod: "/XSync/DpWorkOnBlockCtModGRPC", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(XSyncServer).DpWorkOnCtGetGRPC(ctx, req.(*ConnGet)) + return srv.(XSyncServer).DpWorkOnBlockCtModGRPC(ctx, req.(*BlockCtInfoMod)) } return interceptor(ctx, in, info, handler) } @@ -224,24 +160,16 @@ var XSync_ServiceDesc = grpc.ServiceDesc{ HandlerType: (*XSyncServer)(nil), Methods: []grpc.MethodDesc{ { - MethodName: "DpWorkOnBlockCtAddGRPC", - Handler: _XSync_DpWorkOnBlockCtAddGRPC_Handler, - }, - { - MethodName: "DpWorkOnCtAddGRPC", - Handler: _XSync_DpWorkOnCtAddGRPC_Handler, - }, - { - MethodName: "DpWorkOnBlockCtDelGRPC", - Handler: _XSync_DpWorkOnBlockCtDelGRPC_Handler, + MethodName: "DpWorkOnCtGetGRPC", + Handler: _XSync_DpWorkOnCtGetGRPC_Handler, }, { - MethodName: "DpWorkOnCtDelGRPC", - Handler: _XSync_DpWorkOnCtDelGRPC_Handler, + MethodName: "DpWorkOnCtModGRPC", + Handler: _XSync_DpWorkOnCtModGRPC_Handler, }, { - MethodName: "DpWorkOnCtGetGRPC", - Handler: _XSync_DpWorkOnCtGetGRPC_Handler, + MethodName: "DpWorkOnBlockCtModGRPC", + Handler: _XSync_DpWorkOnBlockCtModGRPC_Handler, }, }, Streams: []grpc.StreamDesc{}, diff --git a/loxinet/xsync_server.go b/loxinet/xsync_server.go index 4f50b69e5..65db20419 100644 --- a/loxinet/xsync_server.go +++ b/loxinet/xsync_server.go @@ -18,16 +18,18 @@ package loxinet import ( "context" - "encoding/json" "errors" "fmt" "net" + "io" + "net/rpc" + "net/http" + "runtime/debug" + "google.golang.org/grpc" - "github.com/golang/protobuf/ptypes/wrappers" tk "github.com/loxilb-io/loxilib" - "google.golang.org/grpc" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/anypb" + opts "github.com/loxilb-io/loxilb/options" + ) // DpWorkOnBlockCtAdd - Add block CT entries from remote goRPC client @@ -38,7 +40,8 @@ func (xs *XSync) DpWorkOnBlockCtAdd(blockCtis []DpCtInfo, ret *int) error { *ret = 0 - mh.dp.DpHooks.DpGetLock() + mh.dp.MapMtx.Lock() + defer mh.dp.MapMtx.Unlock() for _, cti := range blockCtis { @@ -49,8 +52,6 @@ func (xs *XSync) DpWorkOnBlockCtAdd(blockCtis []DpCtInfo, ret *int) error { } } - mh.dp.DpHooks.DpRelLock() - return nil } @@ -61,6 +62,10 @@ func (xs *XSync) DpWorkOnBlockCtDelete(blockCtis []DpCtInfo, ret *int) error { } *ret = 0 + + mh.dp.MapMtx.Lock() + defer mh.dp.MapMtx.Unlock() + for _, cti := range blockCtis { tk.LogIt(tk.LogDebug, "RPC - Block CT Del %s\n", cti.Key()) @@ -105,6 +110,7 @@ func (xs *XSync) DpWorkOnCtAdd(cti DpCtInfo, ret *int) error { } tk.LogIt(tk.LogDebug, "RPC - CT Add %s\n", cti.Key()) + r := mh.dp.DpHooks.DpCtAdd(&cti) *ret = r return nil @@ -128,7 +134,7 @@ func (xs *XSync) DpWorkOnCtGet(async int, ret *int) error { } // Most likely need to reset reverse rpc channel - //mh.dp.DpXsyncRPCReset() + mh.dp.DpXsyncRPCReset() tk.LogIt(tk.LogDebug, "RPC - CT Get %d\n", async) mh.dp.DpHooks.DpCtGetAsync() @@ -137,132 +143,65 @@ func (xs *XSync) DpWorkOnCtGet(async int, ret *int) error { return nil } -func (xs *XSync) DpWorkOnBlockCtAddGRPC(ctx context.Context, m *ConnInfo) (*XSyncReply, error) { - var value interface{} +func (xs *XSync) DpWorkOnCtGetGRPC(ctx context.Context, m *ConnGet) (*XSyncReply, error) { + var resp int - var cti DpCtInfo - var ctis []DpCtInfo - - bytesValue := &wrappers.BytesValue{} - anyValue := m.Cti - - err := anypb.UnmarshalTo(anyValue, bytesValue, proto.UnmarshalOptions{}) - if err != nil { - return &XSyncReply{Response: -1}, err - } else { - uErr := json.Unmarshal(bytesValue.Value, &value) - if uErr != nil { - return &XSyncReply{Response: -1}, uErr - } - //tk.LogIt(tk.LogDebug, "RPC - BLOCK CT ADD : %T : %v\n", value, value) - blk := value.([]interface{}) - for i := range blk { - //tk.LogIt(tk.LogDebug, "BLOCK CT %d: %v\n", i, blk[i]) - jsonData, _:= json.Marshal(blk[i].(map[string]interface{})) - //tk.LogIt(tk.LogDebug, "BLOCK CT json %d: %v\n", i, jsonData) - json.Unmarshal(jsonData, &cti) - ctis = append(ctis, cti) - - } - } - - xs.DpWorkOnBlockCtAdd(ctis, &resp) - return &XSyncReply{Response: int32(resp)}, nil + err := xs.DpWorkOnCtGet(int(m.Async), &resp) + + return &XSyncReply{Response: int32(resp)}, err } -func (xs *XSync) DpWorkOnCtAddGRPC(ctx context.Context, m *ConnInfo) (*XSyncReply, error) { - var value interface{} - var resp int - var cti DpCtInfo - bytesValue := &wrappers.BytesValue{} - anyValue := m.Cti - - err := anypb.UnmarshalTo(anyValue, bytesValue, proto.UnmarshalOptions{}) - if err != nil { - return &XSyncReply{Response: -1}, err - } else { - uErr := json.Unmarshal(bytesValue.Value, &value) - if uErr != nil { - return &XSyncReply{Response: -1}, uErr - } - jsonData, _:= json.Marshal( value.(map[string]interface{})) - json.Unmarshal(jsonData, &cti) +func (ci *CtInfo) ConvertToDpCtInfo() (DpCtInfo) { + + cti := DpCtInfo { + DIP:ci.Dip, SIP:ci.Sip, + Dport: uint16(ci.Dport), Sport: uint16(ci.Sport), + Proto: ci.Proto, CState: ci.Cstate, CAct: ci.Cact, CI:ci.Ci, + Packets: uint64(ci.Packets), Bytes: uint64(ci.Bytes), Deleted: int(ci.Deleted), + PKey: ci.Pkey, PVal: ci.Pval, + XSync: ci.Xsync, ServiceIP: ci.Serviceip, ServProto: ci.Servproto, + L4ServPort: uint16(ci.L4Servport), BlockNum: uint16(ci.Blocknum), } - //tk.LogIt(tk.LogDebug, "RPC - CT ADD : %v\n", cti) - - xs.DpWorkOnCtAdd(cti, &resp) - //tk.LogIt(tk.LogDebug, "RPC - CT ADD Server reply: %v\n", resp) - return &XSyncReply{Response: int32(resp)}, nil + return cti } -func (xs *XSync) DpWorkOnBlockCtDelGRPC(ctx context.Context, m *ConnInfo) (*XSyncReply, error) { - var value interface{} - var resp int - var cti DpCtInfo +func (xs *XSync) DpWorkOnBlockCtModGRPC(ctx context.Context, m *BlockCtInfoMod) (*XSyncReply, error) { var ctis []DpCtInfo - - bytesValue := &wrappers.BytesValue{} - anyValue := m.Cti - - err := anypb.UnmarshalTo(anyValue, bytesValue, proto.UnmarshalOptions{}) - if err != nil { - return &XSyncReply{Response: -1}, err - } else { - uErr := json.Unmarshal(bytesValue.Value, &value) - if uErr != nil { - return &XSyncReply{Response: -1}, uErr - } - //tk.LogIt(tk.LogDebug, "RPC - BLOCK CT DEL : %T : %v\n", value, value) - blk := value.([]interface{}) - for i := range blk { - //tk.LogIt(tk.LogDebug, "BLOCK CT %d: %v\n", i, blk[i]) - jsonData, _:= json.Marshal(blk[i].(map[string]interface{})) - //tk.LogIt(tk.LogDebug, "BLOCK CT json %d: %v\n", i, jsonData) - json.Unmarshal(jsonData, &cti) - ctis = append(ctis, cti) - } - } - - xs.DpWorkOnBlockCtDelete(ctis, &resp) - return &XSyncReply{Response: int32(resp)}, nil -} - -func (xs *XSync) DpWorkOnCtDelGRPC(ctx context.Context, m *ConnInfo) (*XSyncReply, error) { - var value interface{} var resp int - var cti DpCtInfo - - bytesValue := &wrappers.BytesValue{} - anyValue := m.Cti + var err error - err := anypb.UnmarshalTo(anyValue, bytesValue, proto.UnmarshalOptions{}) - if err != nil { - return &XSyncReply{Response: -1}, err + for _, ci := range m.Ct { + cti := ci.ConvertToDpCtInfo() + ctis = append(ctis, cti) + } + if m.Add { + err = xs.DpWorkOnBlockCtAdd(ctis, &resp) } else { - uErr := json.Unmarshal(bytesValue.Value, &value) - if uErr != nil { - return &XSyncReply{Response: -1}, uErr - } - jsonData, _:= json.Marshal( value.(map[string]interface{})) - json.Unmarshal(jsonData, &cti) + err = xs.DpWorkOnBlockCtDelete(ctis, &resp) } - - xs.DpWorkOnCtDelete(cti, &resp) - return &XSyncReply{Response: int32(resp)}, nil + return &XSyncReply{Response: int32(resp)}, err } -func (xs *XSync) DpWorkOnCtGetGRPC(ctx context.Context, m *ConnGet) (*XSyncReply, error) { +func (xs *XSync) DpWorkOnCtModGRPC(ctx context.Context, m *CtInfoMod) (*XSyncReply, error) { var resp int - err := xs.DpWorkOnCtGet(int(m.Async), &resp) + var err error + + ci := m.Ct + cti := ci.ConvertToDpCtInfo() + if m.Add { + err = xs.DpWorkOnCtAdd(cti, &resp) + } else { + err = xs.DpWorkOnCtDelete(cti, &resp) + } return &XSyncReply{Response: int32(resp)}, err } func (xs *XSync) mustEmbedUnimplementedXSyncServer() {} func startxSyncGRPCServer() { - lis, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", XSyncPortGRPC)) + lis, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", XSyncPort)) if err != nil { tk.LogIt(tk.LogEmerg, "gRPC - Server Start Error\n") return @@ -272,4 +211,43 @@ func startxSyncGRPCServer() { RegisterXSyncServer(grpcServer, &s) tk.LogIt(tk.LogNotice, "*******************gRPC - Server Started*****************\n") grpcServer.Serve(lis) -} \ No newline at end of file +} + +// LoxiXsyncMain - State Sync subsystem init +func LoxiXsyncMain(mode string) { + if opts.Opts.ClusterNodes == "none" { + return + } + + // Stack trace logger + defer func() { + if e := recover(); e != nil { + if mh.logger != nil { + tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack()) + } + } + }() + if mode == "netrpc" { + for { + rpcObj := new(XSync) + err := rpc.Register(rpcObj) + if err != nil { + panic("Failed to register rpc") + } + + rpc.HandleHTTP() + + http.HandleFunc("/", func(res http.ResponseWriter, req *http.Request) { + io.WriteString(res, "loxilb-xsync\n") + }) + + listener := fmt.Sprintf(":%d", XSyncPort) + err = http.ListenAndServe(listener, nil) + if err != nil { + panic("Failed to rpc-listen") + } + } + } else { + go startxSyncGRPCServer() + } +} diff --git a/main.go b/main.go index dc6e33d28..709091794 100644 --- a/main.go +++ b/main.go @@ -43,7 +43,7 @@ func main() { os.Exit(0) } - go ln.LoxiXsyncMain() + go ln.LoxiXsyncMain(opts.Opts.Rpc) // Need some time for RPC Handler to be up time.Sleep(2 * time.Second) diff --git a/options/options.go b/options/options.go index 826587afe..875c4c345 100644 --- a/options/options.go +++ b/options/options.go @@ -27,4 +27,5 @@ var Opts struct { EgrHooks bool `long:"egr-hooks" description:"Enable eBPF egress hooks(experimental)"` BgpPeerMode bool `short:"r" long:"peer" description:"Run loxilb with goBGP only, no Datapath"` BlackList string `long:"blacklist" description:"Regex string of blacklisted ports" default:"none"` + Rpc string `long:"rpc" description:"RPC mode for syncing - netrpc or grpc" default:"netrpc"` } From 0aebea5625c55c0a9a2733abe3b71fb0fb8c1803 Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Mon, 4 Sep 2023 10:48:48 +0900 Subject: [PATCH 6/8] xsync grpc optimizations --- loxinet/dpbroker.go | 5 ++--- loxinet/dpebpf_linux.go | 3 --- loxinet/xsync_server.go | 6 ------ 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/loxinet/dpbroker.go b/loxinet/dpbroker.go index 2eca9b388..b96e10d9b 100644 --- a/loxinet/dpbroker.go +++ b/loxinet/dpbroker.go @@ -429,7 +429,6 @@ type DpH struct { ToFinCh chan int DpHooks DpHookInterface SyncMtx sync.RWMutex - MapMtx sync.RWMutex Peers []DpPeer RPC *XSync Remotes []XSync @@ -597,10 +596,10 @@ func callGRPC(client XSyncClient, rpcCallStr string, args interface{}, reply *in if err != nil { *reply = -1 - tk.LogIt(tk.LogInfo, "XSync %s reply - %v[NOK]\n", rpcCallStr, err.Error()) + tk.LogIt(tk.LogError, "XSync %s reply - %v[NOK]\n", rpcCallStr, err.Error()) } else if xreply != nil { *reply = int(xreply.Response) - tk.LogIt(tk.LogInfo, "XSync %s peer reply - %d\n", rpcCallStr, *reply) + tk.LogIt(tk.LogDebug, "XSync %s peer reply - %d\n", rpcCallStr, *reply) } return err } diff --git a/loxinet/dpebpf_linux.go b/loxinet/dpebpf_linux.go index 79f9fc7a0..3d78c3ee6 100644 --- a/loxinet/dpebpf_linux.go +++ b/loxinet/dpebpf_linux.go @@ -187,11 +187,8 @@ func dpEbpfTicker() { // Age any entries related to Conntrack /* No need to fetch all stats in this fashion */ //C.llb_collect_map_stats(C.int(C.LL_DP_CT_STATS_MAP)) - mh.dp.MapMtx.Lock() /* Per entry stats will be fetched in C.ll_ct_map_ent_has_aged */ C.llb_age_map_entries(C.LL_DP_CT_MAP) - mh.dp.MapMtx.Unlock() - C.llb_age_map_entries(C.LL_DP_FCV4_MAP) // This means around 10s from start diff --git a/loxinet/xsync_server.go b/loxinet/xsync_server.go index 65db20419..2ea49fdda 100644 --- a/loxinet/xsync_server.go +++ b/loxinet/xsync_server.go @@ -40,9 +40,6 @@ func (xs *XSync) DpWorkOnBlockCtAdd(blockCtis []DpCtInfo, ret *int) error { *ret = 0 - mh.dp.MapMtx.Lock() - defer mh.dp.MapMtx.Unlock() - for _, cti := range blockCtis { tk.LogIt(tk.LogDebug, "RPC - Block CT Add %s\n", cti.Key()) @@ -63,9 +60,6 @@ func (xs *XSync) DpWorkOnBlockCtDelete(blockCtis []DpCtInfo, ret *int) error { *ret = 0 - mh.dp.MapMtx.Lock() - defer mh.dp.MapMtx.Unlock() - for _, cti := range blockCtis { tk.LogIt(tk.LogDebug, "RPC - Block CT Del %s\n", cti.Key()) From 6dbef7ccc8ff83855c336c569034239c4b0fe5b5 Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Tue, 5 Sep 2023 14:15:21 +0900 Subject: [PATCH 7/8] xsync hooks added --- loxinet/dpbroker.go | 280 ++++++---------------------------------- loxinet/dpebpf_linux.go | 2 +- loxinet/xsync_server.go | 33 +++-- 3 files changed, 56 insertions(+), 259 deletions(-) diff --git a/loxinet/dpbroker.go b/loxinet/dpbroker.go index b96e10d9b..3524cc6c0 100644 --- a/loxinet/dpbroker.go +++ b/loxinet/dpbroker.go @@ -17,22 +17,14 @@ package loxinet import ( - "bufio" - "context" - "errors" "fmt" - "io" "net" - "net/http" - "net/rpc" "runtime/debug" "sync" "time" cmn "github.com/loxilb-io/loxilb/common" tk "github.com/loxilb-io/loxilib" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) // man names constants @@ -82,9 +74,9 @@ const ( // maximum dp work queue lengths const ( - DpWorkQLen = 1024 - XSyncPort = 22222 - DpTiVal = 20 + DpWorkQLen = 1024 + XSyncPort = 22222 + DpTiVal = 20 ) // MirrDpWorkQ - work queue entry for mirror operation @@ -326,12 +318,20 @@ const ( RPCTypeGRPC ) +type RPCHookInterface interface { + RPCConnect(*DpPeer) int + RPCClose(*DpPeer) int + RPCReset(*DpPeer) int + RPCSend(*DpPeer, string, any) (int, error) +} + // XSync - Remote sync peer information type XSync struct { RemoteID int RPCState bool // For peer to peer RPC - RPCType int + RPCType int + RPCHooks RPCHookInterface } // UlClDpWorkQ - work queue entry for ul-cl filter related operation @@ -410,11 +410,6 @@ type DpHookInterface interface { DpRelLock() } -type GRPCClient struct { - conn *grpc.ClientConn - xclient XSyncClient -} - // DpPeer - Remote DP Peer information type DpPeer struct { Peer net.IP @@ -434,82 +429,13 @@ type DpH struct { Remotes []XSync } -// dialHTTPPath connects to an HTTP RPC server -// at the specified network address and path. -// This is based on rpc package's DialHTTPPath but with added timeout -func dialHTTPPath(network, address, path string) (*rpc.Client, error) { - var connected = "200 Connected to Go RPC" - timeOut := 2 * time.Second - - conn, err := net.DialTimeout(network, address, timeOut) - if err != nil { - return nil, err - } - io.WriteString(conn, "CONNECT "+path+" HTTP/1.0\n\n") - - // Require successful HTTP response - // before switching to RPC protocol. - resp, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{Method: "CONNECT"}) - if err == nil && resp.Status == connected { - return rpc.NewClient(conn), nil - } - if err == nil { - err = errors.New("unexpected HTTP response: " + resp.Status) - } - conn.Close() - return nil, &net.OpError{ - Op: "dial-http", - Net: network + " " + address, - Addr: nil, - Err: err, - } -} - // DpXsyncRPCReset - Routine to reset Sunc RPC Client connections func (dp *DpH) DpXsyncRPCReset() int { - var err error dp.SyncMtx.Lock() defer dp.SyncMtx.Unlock() for idx := range mh.dp.Peers { pe := &mh.dp.Peers[idx] - if dp.RPC.RPCType == RPCTypeNetRPC { - cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) - client, ok := pe.Client.(*rpc.Client) - if ok && client != nil { - client.Close() - pe.Client = nil - } - if pe.Client == nil { - pe.Client, err = dialHTTPPath("tcp", cStr, rpc.DefaultRPCPath) - if pe.Client == nil { - tk.LogIt(tk.LogInfo, "Failed to dial xsync goRPC pair: %v", err) - return -1 - } - tk.LogIt(tk.LogInfo, "XSync goRPC - %s :Reset\n", cStr) - } - } else { - var opts []grpc.DialOption - var cinfo GRPCClient - client, ok := pe.Client.(GRPCClient) - if ok { - client.conn.Close() - pe.Client = nil - } - - if pe.Client == nil { - cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) - opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) - cinfo.conn, err = grpc.Dial(cStr, opts...) - if err != nil { - tk.LogIt(tk.LogInfo, "Failed to dial xsync gRPC pair: %v", err) - return -1 - } - - cinfo.xclient = NewXSyncClient(cinfo.conn) - pe.Client = cinfo - tk.LogIt(tk.LogInfo, "XSync gRPC - %s :Reset\n", cStr) - } - } + dp.RPC.RPCHooks.RPCReset(pe) } return 0 } @@ -537,79 +463,11 @@ func (dp *DpH) WaitXsyncReady(who string) { } } -func (ci *DpCtInfo) ConvertToCtInfo(c *CtInfo) { - c.Dip = ci.DIP - c.Sip = ci.SIP - c.Dport = int32(ci.Dport) - c.Sport = int32(ci.Sport) - c.Proto = ci.Proto - c.Cstate = ci.CState - c.Cact = ci.CAct - c.Ci = ci.CI - c.Packets = int64(ci.Packets) - c.Bytes = int64(ci.Bytes) - c.Deleted = int32(ci.Deleted) - c.Pkey = ci.PKey - c.Pval = ci.PVal - c.Xsync = ci.XSync - c.Serviceip = ci.ServiceIP - c.Servproto = ci.ServProto - c.L4Servport = int32(ci.L4ServPort) - c.Blocknum = int32(ci.BlockNum) -} - -func callGRPC(client XSyncClient, rpcCallStr string, args interface{}, reply *int) error { - var err error - var xreply *XSyncReply - var ctis []*CtInfo - var ct *CtInfo - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - if (rpcCallStr == "XSync.DpWorkOnBlockCtAdd") || - (rpcCallStr == "XSync.DpWorkOnBlockCtDelete") { - blkCtis := args.([]DpCtInfo) - ctis = make([]*CtInfo, len(blkCtis)) - for i, c := range blkCtis { - ctis[i] = &CtInfo{} - c.ConvertToCtInfo(ctis[i]) - } - } else if (rpcCallStr == "XSync.DpWorkOnCtAdd") || - (rpcCallStr == "XSync.DpWorkOnCtDelete") { - c := args.(DpCtInfo) - ct = &CtInfo{} - c.ConvertToCtInfo(ct) - } - - if rpcCallStr == "XSync.DpWorkOnBlockCtAdd" { - xreply, err = client.DpWorkOnBlockCtModGRPC(ctx, &BlockCtInfoMod{Add: true, Ct: ctis}) - } else if rpcCallStr == "XSync.DpWorkOnBlockCtDelete" { - xreply, err = client.DpWorkOnBlockCtModGRPC(ctx, &BlockCtInfoMod{Add: false, Ct: ctis}) - } else if rpcCallStr == "XSync.DpWorkOnCtAdd" { - xreply, err = client.DpWorkOnCtModGRPC(ctx, &CtInfoMod{Add: true, Ct: ct}) - } else if rpcCallStr == "XSync.DpWorkOnCtDelete" { - xreply, err = client.DpWorkOnCtModGRPC(ctx, &CtInfoMod{Add: false, Ct: ct}) - } else if rpcCallStr == "XSync.DpWorkOnCtGet" { - xreply, err = client.DpWorkOnCtGetGRPC(ctx, &ConnGet{Async: args.(int32)}) - } - - if err != nil { - *reply = -1 - tk.LogIt(tk.LogError, "XSync %s reply - %v[NOK]\n", rpcCallStr, err.Error()) - } else if xreply != nil { - *reply = int(xreply.Response) - tk.LogIt(tk.LogDebug, "XSync %s peer reply - %d\n", rpcCallStr, *reply) - } - return err -} - // DpXsyncRPC - Routine for syncing connection information with peers func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { - var reply int + var reply,ret int var err error - timeout := 2 * time.Second dp.SyncMtx.Lock() defer dp.SyncMtx.Unlock() @@ -633,31 +491,10 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { restartRPC: pe := &mh.dp.Peers[idx] if pe.Client == nil { - var err error - if dp.RPC.RPCType == RPCTypeNetRPC { - cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) - pe.Client, err = dialHTTPPath("tcp", cStr, rpc.DefaultRPCPath) - if pe.Client == nil || err != nil { - tk.LogIt(tk.LogInfo, "XSync RPC - %s :Fail(%s)\n", cStr, err) - rpcErr = true - pe.Client = nil - continue - } - tk.LogIt(tk.LogInfo, "XSync goRPC - %s :Connected\n", cStr) - } else { - var opts []grpc.DialOption - var cinfo GRPCClient - opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) - cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) - cinfo.conn, err = grpc.Dial(cStr, opts...) - if err != nil { - tk.LogIt(tk.LogInfo, "Failed to dial xsync gRPC pair: %v", err) - continue - } - - cinfo.xclient = NewXSyncClient(cinfo.conn) - pe.Client = cinfo - tk.LogIt(tk.LogInfo, "XSync gRPC - %s :Connected\n", cStr) + ret = dp.RPC.RPCHooks.RPCConnect(pe) + if ret != 0 { + rpcErr = true + continue } } @@ -681,7 +518,6 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { return -1 } - var call *rpc.Call if op == DpSyncAdd || op == DpSyncDelete || op == DpSyncBcast { if op != DpSyncBcast { if cti == nil && len(blkCti) <= 0 { @@ -700,68 +536,29 @@ func (dp *DpH) DpXsyncRPC(op DpSyncOpT, arg interface{}) int { return 0 } } - if dp.RPC.RPCType == RPCTypeNetRPC { - client, _ := pe.Client.(*rpc.Client) - if cti != nil { - call = client.Go(rpcCallStr, *cti, &reply, make(chan *rpc.Call, 1)) - } else { - call = client.Go(rpcCallStr, blkCti, &reply, make(chan *rpc.Call, 1)) - } + if cti != nil { + reply, err = dp.RPC.RPCHooks.RPCSend(pe, rpcCallStr, *cti) } else { - client, _ := pe.Client.(GRPCClient) - if cti != nil { - err = callGRPC(client.xclient, rpcCallStr, *cti, &reply) - } else { - err = callGRPC(client.xclient, rpcCallStr, blkCti, &reply) - } + reply, err = dp.RPC.RPCHooks.RPCSend(pe, rpcCallStr, blkCti) } } else { async := 1 - if dp.RPC.RPCType == RPCTypeNetRPC { - call = pe.Client.(*rpc.Client).Go(rpcCallStr, async, &reply, make(chan *rpc.Call, 1)) - } else { - err = callGRPC(pe.Client.(GRPCClient).xclient, rpcCallStr, int32(async), &reply) - } + reply, err = dp.RPC.RPCHooks.RPCSend(pe, rpcCallStr, int32(async)) } - if dp.RPC.RPCType == RPCTypeNetRPC { - select { - case <-time.After(timeout): - tk.LogIt(tk.LogError, "rpc call timeout(%v)\n", timeout) - if pe.Client != nil { - if dp.RPC.RPCType == RPCTypeNetRPC { - pe.Client.(*rpc.Client).Close() - } else { - pe.Client.(GRPCClient).conn.Close() - } - } - pe.Client = nil - rpcRetries++ - if rpcRetries < 2 { - goto restartRPC - } - rpcErr = true - case resp := <-call.Done: - if resp != nil && resp.Error != nil { - tk.LogIt(tk.LogError, "rpc call failed(%s)\n", resp.Error) - rpcErr = true - } - } - } else { - if err != nil { - tk.LogIt(tk.LogError, "grpc call failed(%s)\n", err) - rpcErr = true - pe.Client = nil - rpcRetries++ - if rpcRetries < 2 { - goto restartRPC - } - } - if reply != 0 { - tk.LogIt(tk.LogError, "grpc server returned error (%d)\n", reply) - rpcErr = true + if err != nil { + tk.LogIt(tk.LogError, "XSync call failed(%s)\n", err) + rpcErr = true + pe.Client = nil + rpcRetries++ + if rpcRetries < 2 { + goto restartRPC } } + if reply != 0 { + tk.LogIt(tk.LogError, "Xsync server returned error (%d)\n", reply) + rpcErr = true + } } if rpcErr { @@ -781,6 +578,11 @@ func DpBrokerInit(dph DpHookInterface, rpcMode int) *DpH { nDp.RPC = new(XSync) nDp.RPC.RPCType = rpcMode + if (rpcMode == RPCTypeNetRPC) { + nDp.RPC.RPCHooks = &netRPCClient{} + } else { + nDp.RPC.RPCHooks = &gRPCClient{} + } go DpWorker(nDp, nDp.ToFinCh, nDp.ToDpCh) @@ -925,11 +727,7 @@ func (dp *DpH) DpWorkOnPeerOp(pWq *PeerDpWorkQ) DpRetT { pe := &dp.Peers[idx] if pe.Peer.Equal(pWq.PeerIP) { if pe.Client != nil { - if dp.RPC.RPCType == RPCTypeNetRPC { - pe.Client.(*rpc.Client).Close() - } else { - pe.Client.(GRPCClient).conn.Close() - } + dp.RPC.RPCHooks.RPCClose(pe) } dp.Peers = append(dp.Peers[:idx], dp.Peers[idx+1:]...) tk.LogIt(tk.LogInfo, "Deleted cluster-peer %s\n", pWq.PeerIP.String()) diff --git a/loxinet/dpebpf_linux.go b/loxinet/dpebpf_linux.go index 3d78c3ee6..bf7c3397a 100644 --- a/loxinet/dpebpf_linux.go +++ b/loxinet/dpebpf_linux.go @@ -453,7 +453,7 @@ func (e *DpEbpfH) DpPortPropMod(w *PortDpWorkQ) int { lRet := e.loadEbpfPgm(w.LoadEbpf) if lRet != 0 { tk.LogIt(tk.LogError, "ebpf load - %d error\n", w.PortNum) - return EbpfErrEbpfLoad + syscall.Exit(1) } } data := new(intfMapDat) diff --git a/loxinet/xsync_server.go b/loxinet/xsync_server.go index 2ea49fdda..d71e607a6 100644 --- a/loxinet/xsync_server.go +++ b/loxinet/xsync_server.go @@ -20,16 +20,15 @@ import ( "context" "errors" "fmt" - "net" "io" - "net/rpc" + "net" "net/http" + "net/rpc" "runtime/debug" - "google.golang.org/grpc" - tk "github.com/loxilb-io/loxilib" + "google.golang.org/grpc" opts "github.com/loxilb-io/loxilb/options" - + tk "github.com/loxilb-io/loxilib" ) // DpWorkOnBlockCtAdd - Add block CT entries from remote goRPC client @@ -59,7 +58,7 @@ func (xs *XSync) DpWorkOnBlockCtDelete(blockCtis []DpCtInfo, ret *int) error { } *ret = 0 - + for _, cti := range blockCtis { tk.LogIt(tk.LogDebug, "RPC - Block CT Del %s\n", cti.Key()) @@ -104,7 +103,7 @@ func (xs *XSync) DpWorkOnCtAdd(cti DpCtInfo, ret *int) error { } tk.LogIt(tk.LogDebug, "RPC - CT Add %s\n", cti.Key()) - + r := mh.dp.DpHooks.DpCtAdd(&cti) *ret = r return nil @@ -138,22 +137,22 @@ func (xs *XSync) DpWorkOnCtGet(async int, ret *int) error { } func (xs *XSync) DpWorkOnCtGetGRPC(ctx context.Context, m *ConnGet) (*XSyncReply, error) { - + var resp int err := xs.DpWorkOnCtGet(int(m.Async), &resp) - + return &XSyncReply{Response: int32(resp)}, err } -func (ci *CtInfo) ConvertToDpCtInfo() (DpCtInfo) { - - cti := DpCtInfo { - DIP:ci.Dip, SIP:ci.Sip, +func (ci *CtInfo) ConvertToDpCtInfo() DpCtInfo { + + cti := DpCtInfo{ + DIP: ci.Dip, SIP: ci.Sip, Dport: uint16(ci.Dport), Sport: uint16(ci.Sport), - Proto: ci.Proto, CState: ci.Cstate, CAct: ci.Cact, CI:ci.Ci, + Proto: ci.Proto, CState: ci.Cstate, CAct: ci.Cact, CI: ci.Ci, Packets: uint64(ci.Packets), Bytes: uint64(ci.Bytes), Deleted: int(ci.Deleted), PKey: ci.Pkey, PVal: ci.Pval, - XSync: ci.Xsync, ServiceIP: ci.Serviceip, ServProto: ci.Servproto, + XSync: ci.Xsync, ServiceIP: ci.Serviceip, ServProto: ci.Servproto, L4ServPort: uint16(ci.L4Servport), BlockNum: uint16(ci.Blocknum), } return cti @@ -177,13 +176,13 @@ func (xs *XSync) DpWorkOnBlockCtModGRPC(ctx context.Context, m *BlockCtInfoMod) } func (xs *XSync) DpWorkOnCtModGRPC(ctx context.Context, m *CtInfoMod) (*XSyncReply, error) { - + var resp int var err error ci := m.Ct cti := ci.ConvertToDpCtInfo() - + if m.Add { err = xs.DpWorkOnCtAdd(cti, &resp) } else { From e13ae6701e817902b68cee2f02d16e47c9ae1363 Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Tue, 5 Sep 2023 14:18:03 +0900 Subject: [PATCH 8/8] xsync_client file added --- loxinet/xsync_client.go | 263 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 loxinet/xsync_client.go diff --git a/loxinet/xsync_client.go b/loxinet/xsync_client.go new file mode 100644 index 000000000..32d7cc8fd --- /dev/null +++ b/loxinet/xsync_client.go @@ -0,0 +1,263 @@ +/* + * Copyright (c) 2022 NetLOX Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loxinet + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/rpc" + "time" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + tk "github.com/loxilb-io/loxilib" +) + +type netRPCClient struct { + client *rpc.Client +} + +type gRPCClient struct { + conn *grpc.ClientConn + xclient XSyncClient +} + +// dialHTTPPath connects to an HTTP RPC server +// at the specified network address and path. +// This is based on rpc package's DialHTTPPath but with added timeout +func dialHTTPPath(network, address, path string) (*rpc.Client, error) { + var connected = "200 Connected to Go RPC" + timeOut := 2 * time.Second + + conn, err := net.DialTimeout(network, address, timeOut) + if err != nil { + return nil, err + } + io.WriteString(conn, "CONNECT "+path+" HTTP/1.0\n\n") + + // Require successful HTTP response + // before switching to RPC protocol. + resp, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{Method: "CONNECT"}) + if err == nil && resp.Status == connected { + return rpc.NewClient(conn), nil + } + if err == nil { + err = errors.New("unexpected HTTP response: " + resp.Status) + } + conn.Close() + return nil, &net.OpError{ + Op: "dial-http", + Net: network + " " + address, + Addr: nil, + Err: err, + } +} + +func netRPCConnect(pe *DpPeer) int { + cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) + client, err := dialHTTPPath("tcp", cStr, rpc.DefaultRPCPath) + if client == nil || err != nil { + tk.LogIt(tk.LogInfo, "XSync netRPC Connect - %s :Fail(%s)\n", cStr, err) + pe.Client = nil + return -1 + } + pe.Client = client + tk.LogIt(tk.LogInfo, "XSync netRPC - %s :Connected\n", cStr) + return 0 +} + +func (*netRPCClient) RPCConnect(pe *DpPeer) int { + return netRPCConnect(pe) +} + +func (*netRPCClient) RPCReset(pe *DpPeer) int { + cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) + client, ok := pe.Client.(*rpc.Client) + if ok && client != nil { + client.Close() + pe.Client = nil + } + if pe.Client == nil { + tk.LogIt(tk.LogInfo, "XSync netRPC - %s :Reset\n", cStr) + return netRPCConnect(pe) + } + return 0 +} + +func (*netRPCClient) RPCClose(pe *DpPeer) int { + if pe.Client != nil { + pe.Client.(*rpc.Client).Close() + } + pe.Client = nil + return 0 +} + +func (*netRPCClient) RPCSend(pe *DpPeer, rpcCallStr string, args any) (int, error) { + var reply int + client, _ := pe.Client.(*rpc.Client) + timeout := 2 * time.Second + call := client.Go(rpcCallStr, args, &reply, make(chan *rpc.Call, 1)) + select { + case <-time.After(timeout): + tk.LogIt(tk.LogError, "netRPC call timeout(%v)\n", timeout) + if pe.Client != nil { + pe.Client.(*rpc.Client).Close() + } + pe.Client = nil + + return reply, errors.New("netrpc call timeout") + case resp := <-call.Done: + if resp != nil && resp.Error != nil { + tk.LogIt(tk.LogError, "netRPC send failed(%s)\n", resp.Error) + return reply, resp.Error + } + } + return reply, nil +} + +func gRPCConnect(pe *DpPeer) int { + var err error + var opts []grpc.DialOption + var cinfo gRPCClient + cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) + + timeOut := 2 * time.Second + + _, err = net.DialTimeout("tcp", cStr, timeOut) + if err != nil { + tk.LogIt(tk.LogInfo, "Failed to dial xsync pair(%s): %v\n", cStr, err) + return -1 + } + + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + cinfo.conn, err = grpc.Dial(cStr, opts...) + + if cinfo.conn == nil || err != nil { + tk.LogIt(tk.LogInfo, "Failed to dial xsync gRPC pair: %v\n", err) + return -1 + } + + cinfo.xclient = NewXSyncClient(cinfo.conn) + pe.Client = cinfo + tk.LogIt(tk.LogInfo, "XSync gRPC - %s :Connected\n", cStr) + return 0 +} + +func (*gRPCClient) RPCConnect(pe *DpPeer) int { + return gRPCConnect(pe) +} + +func (*gRPCClient) RPCReset(pe *DpPeer) int { + cStr := fmt.Sprintf("%s:%d", pe.Peer.String(), XSyncPort) + client, ok := pe.Client.(gRPCClient) + if ok { + client.conn.Close() + pe.Client = nil + } + + if pe.Client == nil { + tk.LogIt(tk.LogInfo, "XSync gRPC - %s :Reset\n", cStr) + return gRPCConnect(pe) + } + return 0 +} + +func (*gRPCClient) RPCClose(pe *DpPeer) int { + if pe.Client != nil { + pe.Client.(gRPCClient).conn.Close() + } + pe.Client = nil + return 0 +} + +func (ci *DpCtInfo) ConvertToCtInfo(c *CtInfo) { + c.Dip = ci.DIP + c.Sip = ci.SIP + c.Dport = int32(ci.Dport) + c.Sport = int32(ci.Sport) + c.Proto = ci.Proto + c.Cstate = ci.CState + c.Cact = ci.CAct + c.Ci = ci.CI + c.Packets = int64(ci.Packets) + c.Bytes = int64(ci.Bytes) + c.Deleted = int32(ci.Deleted) + c.Pkey = ci.PKey + c.Pval = ci.PVal + c.Xsync = ci.XSync + c.Serviceip = ci.ServiceIP + c.Servproto = ci.ServProto + c.L4Servport = int32(ci.L4ServPort) + c.Blocknum = int32(ci.BlockNum) +} + +func callGRPC(client XSyncClient, rpcCallStr string, args interface{}, reply *int) error { + var err error + var xreply *XSyncReply + var ctis []*CtInfo + var ct *CtInfo + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if (rpcCallStr == "XSync.DpWorkOnBlockCtAdd") || + (rpcCallStr == "XSync.DpWorkOnBlockCtDelete") { + blkCtis := args.([]DpCtInfo) + ctis = make([]*CtInfo, len(blkCtis)) + for i, c := range blkCtis { + ctis[i] = &CtInfo{} + c.ConvertToCtInfo(ctis[i]) + } + } else if (rpcCallStr == "XSync.DpWorkOnCtAdd") || + (rpcCallStr == "XSync.DpWorkOnCtDelete") { + c := args.(DpCtInfo) + ct = &CtInfo{} + c.ConvertToCtInfo(ct) + } + + if rpcCallStr == "XSync.DpWorkOnBlockCtAdd" { + xreply, err = client.DpWorkOnBlockCtModGRPC(ctx, &BlockCtInfoMod{Add: true, Ct: ctis}) + } else if rpcCallStr == "XSync.DpWorkOnBlockCtDelete" { + xreply, err = client.DpWorkOnBlockCtModGRPC(ctx, &BlockCtInfoMod{Add: false, Ct: ctis}) + } else if rpcCallStr == "XSync.DpWorkOnCtAdd" { + xreply, err = client.DpWorkOnCtModGRPC(ctx, &CtInfoMod{Add: true, Ct: ct}) + } else if rpcCallStr == "XSync.DpWorkOnCtDelete" { + xreply, err = client.DpWorkOnCtModGRPC(ctx, &CtInfoMod{Add: false, Ct: ct}) + } else if rpcCallStr == "XSync.DpWorkOnCtGet" { + xreply, err = client.DpWorkOnCtGetGRPC(ctx, &ConnGet{Async: args.(int32)}) + } + + if err != nil { + *reply = -1 + tk.LogIt(tk.LogError, "XSync %s reply - %v[NOK]\n", rpcCallStr, err.Error()) + } else if xreply != nil { + *reply = int(xreply.Response) + tk.LogIt(tk.LogDebug, "XSync %s peer reply - %d\n", rpcCallStr, *reply) + } + return err +} + +func (*gRPCClient) RPCSend(pe *DpPeer, rpcCallStr string, args any) (int, error) { + var reply int + err := callGRPC(pe.Client.(gRPCClient).xclient, rpcCallStr, args, &reply) + + return reply,err +}