From ea2115f9938a49fcdacd32555206d7b21cb6381b Mon Sep 17 00:00:00 2001
From: Mahdi Seyedan <mahdi.se@yahoo.com>
Date: Thu, 23 May 2024 16:11:38 +0330
Subject: [PATCH 01/10] feat: add weaviate service

---
 doc/weaviate.md       |  46 +++++++++++++++++++
 nix/default.nix       |   1 +
 nix/weaviate.nix      | 101 ++++++++++++++++++++++++++++++++++++++++++
 nix/weaviate_test.nix |  32 +++++++++++++
 test/flake.nix        |   1 +
 5 files changed, 181 insertions(+)
 create mode 100644 doc/weaviate.md
 create mode 100644 nix/weaviate.nix
 create mode 100644 nix/weaviate_test.nix

diff --git a/doc/weaviate.md b/doc/weaviate.md
new file mode 100644
index 00000000..3d5e341c
--- /dev/null
+++ b/doc/weaviate.md
@@ -0,0 +1,46 @@
+# Weaviate
+
+{#start}
+
+## Getting started
+
+```nix
+# In `perSystem.process-compose.<name>`
+{
+  services.weaviate."weaviate1".enable = true;
+}
+```
+
+{#tips}
+
+## Tips & Tricks
+
+{#port}
+
+### Use a different port
+
+```nix
+{
+  services.weaviate."weaviate1" = {
+    enable = true;
+    port = 8080;
+  };
+}
+```
+
+{#dataDir}
+
+### Use a different data path
+
+```nix
+{
+  services.weaviate."weaviate1" = {
+    enable = true;
+    settings = {
+      persistence = {
+        dataPath = "./data";
+      };
+    };
+  };
+}
+```
diff --git a/nix/default.nix b/nix/default.nix
index 3051221c..a1fc194f 100644
--- a/nix/default.nix
+++ b/nix/default.nix
@@ -18,5 +18,6 @@ in
     ./pgadmin.nix
     ./cassandra.nix
     ./tempo.nix
+    ./weaviate.nix
   ];
 }
diff --git a/nix/weaviate.nix b/nix/weaviate.nix
new file mode 100644
index 00000000..2a617eb0
--- /dev/null
+++ b/nix/weaviate.nix
@@ -0,0 +1,101 @@
+{ pkgs, lib, name, config, ... }:
+let
+  inherit (lib) types;
+  format = pkgs.formats.json { };
+in
+{
+  options = {
+    enable = lib.mkEnableOption name;
+
+    package = lib.mkPackageOption pkgs "weaviate" { };
+
+    host = lib.mkOption {
+      type = types.nullOr types.str;
+      default = "0.0.0.0";
+      description = ''
+        The IP to listen on
+      '';
+      example = "0.0.0.0";
+    };
+
+    port = lib.mkOption {
+      type = types.port;
+      default = 8080;
+      description = ''
+        The port to listen on for connections
+      '';
+    };
+
+    settings = lib.mkOption {
+      type = format.type;
+      default = { };
+      description = ''
+        Weaviate configuration.
+      '';
+      example = lib.literalExpression ''
+        {
+          "authentication": {
+            "anonymous_access": {
+              "enabled": true
+            }
+          },
+          "authorization": {
+            "admin_list": {
+              "enabled": false
+            }
+          },
+          "query_defaults": {
+            "limit": 100
+          },
+          "persistence": {
+            "dataPath": "./data"
+          }
+        }
+      '';
+    };
+
+    outputs.settings = lib.mkOption {
+      type = types.deferredModule;
+      internal = true;
+      readOnly = true;
+      default = {
+        processes = {
+          "${name}" =
+            let
+              configFile = format.generate "weaviate.conf.json" config.settings;
+
+              startScript = pkgs.writeShellApplication {
+                name = "start-weaviate";
+                runtimeInputs = [ config.package ];
+                text = ''
+                  exec weaviate --scheme http --host ${config.host} --port ${toString config.port} --config-file ${configFile}
+                '';
+              };
+
+              readyScript = pkgs.writeText "ready.py" ''
+                import weaviate
+                client = weaviate.connect_to_local(port=${toString config.port}, host="${config.host}")
+                client.close()
+              '';
+            in
+            {
+              command = startScript;
+
+              readiness_probe = {
+                exec.command = "${(pkgs.python3.withPackages (p: [ p.weaviate-client ]))}/bin/python ${readyScript}";
+                initial_delay_seconds = 2;
+                period_seconds = 10;
+                timeout_seconds = 4;
+                success_threshold = 1;
+                failure_threshold = 5;
+              };
+              namespace = name;
+
+              # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
+              availability.restart = "on_failure";
+            };
+        };
+      };
+    };
+  };
+}
diff --git a/nix/weaviate_test.nix b/nix/weaviate_test.nix
new file mode 100644
index 00000000..0444d579
--- /dev/null
+++ b/nix/weaviate_test.nix
@@ -0,0 +1,32 @@
+{ pkgs, config, ... }: {
+  services.weaviate."weaviate1".enable = true;
+
+  settings.processes.test =
+    let
+      cfg = config.services.weaviate."weaviate1";
+      testScript = pkgs.writeText "test.py" ''
+        import weaviate
+
+        client = weaviate.connect_to_local(
+            port=${toString cfg.port},
+            host="${cfg.host}"
+        )
+        client.close()
+      '';
+    in
+    {
+      command = pkgs.writeShellApplication {
+        runtimeInputs = [
+          cfg.package
+          (pkgs.python3.withPackages (python-pkgs: [
+            python-pkgs.weaviate-client
+          ]))
+        ];
+        text = ''
+          exec python3 ${testScript}
+        '';
+        name = "weaviate-test";
+      };
+      depends_on."weaviate1".condition = "process_healthy";
+    };
+}
diff --git a/test/flake.nix b/test/flake.nix
index dca431c4..aa9a0cbc 100644
--- a/test/flake.nix
+++ b/test/flake.nix
@@ -47,6 +47,7 @@
             "${inputs.services-flake}/nix/pgadmin_test.nix"
             "${inputs.services-flake}/nix/cassandra_test.nix"
             "${inputs.services-flake}/nix/tempo_test.nix"
+            "${inputs.services-flake}/nix/weaviate_test.nix"
           ]);
       };
     };

From e45010ef99833fb3ec177025853fb1e07e999d4e Mon Sep 17 00:00:00 2001
From: Mahdi Seyedan <mahdi.se@yahoo.com>
Date: Fri, 24 May 2024 16:55:37 +0330
Subject: [PATCH 02/10] fix(weaviate): use environment variables instead of
 config file

---
 nix/weaviate.nix | 47 +++++++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 22 deletions(-)

diff --git a/nix/weaviate.nix b/nix/weaviate.nix
index 2a617eb0..8bf4f391 100644
--- a/nix/weaviate.nix
+++ b/nix/weaviate.nix
@@ -1,7 +1,6 @@
 { pkgs, lib, name, config, ... }:
 let
   inherit (lib) types;
-  format = pkgs.formats.json { };
 in
 {
   options = {
@@ -9,6 +8,12 @@ in
 
     package = lib.mkPackageOption pkgs "weaviate" { };
 
+    dataDir = lib.mkOption {
+      type = types.str;
+      default = "./data";
+      description = "Path to the Weaviate data store";
+    };
+
     host = lib.mkOption {
       type = types.nullOr types.str;
       default = "0.0.0.0";
@@ -26,30 +31,19 @@ in
       '';
     };
 
-    settings = lib.mkOption {
-      type = format.type;
+    envs = lib.mkOption {
+      type = types.attrsOf (types.oneOf [ types.str types.int types.bool (types.listOf types.str) ]);
       default = { };
       description = ''
-        Weaviate configuration.
+        Weaviate environment variables.
       '';
       example = lib.literalExpression ''
         {
-          "authentication": {
-            "anonymous_access": {
-              "enabled": true
-            }
-          },
-          "authorization": {
-            "admin_list": {
-              "enabled": false
-            }
-          },
-          "query_defaults": {
-            "limit": 100
-          },
-          "persistence": {
-            "dataPath": "./data"
-          }
+          AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED = true;
+          QUERY_DEFAULTS_LIMIT = 100;
+          DISABLE_TELEMETRY = true;
+          LIMIT_RESOURCES = true;
+          ENABLE_MODULES = ["text2vec-openai" "generative-openai"];
         }
       '';
     };
@@ -62,13 +56,22 @@ in
         processes = {
           "${name}" =
             let
-              configFile = format.generate "weaviate.conf.json" config.settings;
+              toStr = value:
+                if builtins.isString value then builtins.toJSON value
+                else if builtins.isBool value then (if value then "true" else "false")
+                else if builtins.isList value then builtins.toJSON (lib.concatStringsSep "," value)
+                else if builtins.isInt value then toString value
+                else throw "Unrecognized type";
+
+              exports = (lib.mapAttrsToList (name: value: "export ${name}=${toStr value}") ({ "PERSISTENCE_DATA_PATH" = config.dataDir; }
+                // config.envs));
 
               startScript = pkgs.writeShellApplication {
                 name = "start-weaviate";
                 runtimeInputs = [ config.package ];
                 text = ''
-                  exec weaviate --scheme http --host ${config.host} --port ${toString config.port} --config-file ${configFile}
+                  ${lib.concatStringsSep "\n" exports}
+                  exec weaviate --scheme http --host ${config.host} --port ${toString config.port}
                 '';
               };
 

From a228b5bb8b809d1160a2eecb411c56133fb9577b Mon Sep 17 00:00:00 2001
From: Mahdi Seyedan <mahdi.se@yahoo.com>
Date: Fri, 24 May 2024 17:13:23 +0330
Subject: [PATCH 03/10] refactor(weaviate): better test and readiness_probe

---
 nix/weaviate.nix      | 14 ++++++--------
 nix/weaviate_test.nix | 18 ++----------------
 2 files changed, 8 insertions(+), 24 deletions(-)

diff --git a/nix/weaviate.nix b/nix/weaviate.nix
index 8bf4f391..50d239f7 100644
--- a/nix/weaviate.nix
+++ b/nix/weaviate.nix
@@ -74,19 +74,17 @@ in
                   exec weaviate --scheme http --host ${config.host} --port ${toString config.port}
                 '';
               };
-
-              readyScript = pkgs.writeText "ready.py" ''
-                import weaviate
-                client = weaviate.connect_to_local(port=${toString config.port}, host="${config.host}")
-                client.close()
-              '';
             in
             {
               command = startScript;
 
               readiness_probe = {
-                exec.command = "${(pkgs.python3.withPackages (p: [ p.weaviate-client ]))}/bin/python ${readyScript}";
-                initial_delay_seconds = 2;
+                http_get = {
+                  host = config.host;
+                  port = config.port;
+                  path = "/v1/.well-known/ready";
+                };
+                initial_delay_seconds = 3;
                 period_seconds = 10;
                 timeout_seconds = 4;
                 success_threshold = 1;
diff --git a/nix/weaviate_test.nix b/nix/weaviate_test.nix
index 0444d579..a766c582 100644
--- a/nix/weaviate_test.nix
+++ b/nix/weaviate_test.nix
@@ -4,26 +4,12 @@
   settings.processes.test =
     let
       cfg = config.services.weaviate."weaviate1";
-      testScript = pkgs.writeText "test.py" ''
-        import weaviate
-
-        client = weaviate.connect_to_local(
-            port=${toString cfg.port},
-            host="${cfg.host}"
-        )
-        client.close()
-      '';
     in
     {
       command = pkgs.writeShellApplication {
-        runtimeInputs = [
-          cfg.package
-          (pkgs.python3.withPackages (python-pkgs: [
-            python-pkgs.weaviate-client
-          ]))
-        ];
+        runtimeInputs = [ cfg.package pkgs.curl ];
         text = ''
-          exec python3 ${testScript}
+          curl http://localhost:8080/v1/.well-known/live
         '';
         name = "weaviate-test";
       };

From c48d0481817315dc255e1854c3c02e82bfd71cd2 Mon Sep 17 00:00:00 2001
From: Mahdi Seyedan <mahdi.se@yahoo.com>
Date: Fri, 24 May 2024 21:47:02 +0330
Subject: [PATCH 04/10] fix(weaviate): fix memberlist error in test

---
 nix/weaviate.nix      | 2 +-
 nix/weaviate_test.nix | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/nix/weaviate.nix b/nix/weaviate.nix
index 50d239f7..ebffe4ff 100644
--- a/nix/weaviate.nix
+++ b/nix/weaviate.nix
@@ -16,7 +16,7 @@ in
 
     host = lib.mkOption {
       type = types.nullOr types.str;
-      default = "0.0.0.0";
+      default = "127.0.0.1";
       description = ''
         The IP to listen on
       '';
diff --git a/nix/weaviate_test.nix b/nix/weaviate_test.nix
index a766c582..31d79758 100644
--- a/nix/weaviate_test.nix
+++ b/nix/weaviate_test.nix
@@ -1,5 +1,10 @@
 { pkgs, config, ... }: {
-  services.weaviate."weaviate1".enable = true;
+  services.weaviate."weaviate1" = {
+    enable = true;
+    envs = {
+      CLUSTER_ADVERTISE_ADDR = "127.0.0.1";
+    };
+  };
 
   settings.processes.test =
     let

From 1f5e9277851cbd00f76e9c13583a0a8eff43b84e Mon Sep 17 00:00:00 2001
From: Mahdi Seyedan <mahdi.se@yahoo.com>
Date: Fri, 24 May 2024 22:11:04 +0330
Subject: [PATCH 05/10] docs(weaviate): update doc with new configs

---
 doc/services.md |  1 +
 doc/weaviate.md | 31 ++++++++++++++++++++++++++-----
 2 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/doc/services.md b/doc/services.md
index 648f48ae..1ad3cb6a 100644
--- a/doc/services.md
+++ b/doc/services.md
@@ -21,5 +21,6 @@ short-title: Services
   - [[tempo]]
 - [[prometheus]]#
 - [[cassandra]]#
+- [[weaviate]]#
 
 [gh]: https://github.com/juspay/services-flake/issues/132
diff --git a/doc/weaviate.md b/doc/weaviate.md
index 3d5e341c..c2cfab8c 100644
--- a/doc/weaviate.md
+++ b/doc/weaviate.md
@@ -1,5 +1,9 @@
 # Weaviate
 
+[Weaviate] is an open-source vector database that stores both objects and vectors, allowing for the combination of vector search with structured filtering with the fault tolerance and scalability of a cloud-native database.
+
+[Weaviate]: https://github.com/weaviate/weaviate
+
 {#start}
 
 ## Getting started
@@ -15,6 +19,27 @@
 
 ## Tips & Tricks
 
+{#envs}
+
+### Environment variables
+
+To see list of environment variables, see [this link](https://weaviate.io/developers/weaviate/config-refs/env-vars).
+
+```nix
+{
+  services.weaviate."weaviate1" = {
+    enable = true;
+    envs = {
+      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED = true;
+      QUERY_DEFAULTS_LIMIT = 100;
+      DISABLE_TELEMETRY = true;
+      LIMIT_RESOURCES = true;
+      ENABLE_MODULES = ["text2vec-openai" "generative-openai"];
+    };
+  };
+}
+```
+
 {#port}
 
 ### Use a different port
@@ -36,11 +61,7 @@
 {
   services.weaviate."weaviate1" = {
     enable = true;
-    settings = {
-      persistence = {
-        dataPath = "./data";
-      };
-    };
+    dataDir = "./data";
   };
 }
 ```

From 8ca628cfcf8f1ecfe65f27155d444744b9cd9627 Mon Sep 17 00:00:00 2001
From: Mahdi Seyedan <mahdi.se@yahoo.com>
Date: Sun, 26 May 2024 00:14:30 +0330
Subject: [PATCH 06/10] chore(weaviate): add a comment to explain envs in test

---
 nix/weaviate_test.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nix/weaviate_test.nix b/nix/weaviate_test.nix
index 31d79758..5917b5b4 100644
--- a/nix/weaviate_test.nix
+++ b/nix/weaviate_test.nix
@@ -2,6 +2,7 @@
   services.weaviate."weaviate1" = {
     enable = true;
     envs = {
+      # To fix the problem with nix failing to run weaviate in sandbox mode
       CLUSTER_ADVERTISE_ADDR = "127.0.0.1";
     };
   };

From 067c1e2933c02bfa619a349c4d5b98947110864a Mon Sep 17 00:00:00 2001
From: Mahdi Seyedan <mahdi.se@yahoo.com>
Date: Mon, 27 May 2024 01:35:36 +0330
Subject: [PATCH 07/10] refactor(weaviate): refactor environment to use
 process-compose option

---
 nix/weaviate.nix      | 24 ++++++++++++------------
 nix/weaviate_test.nix |  2 +-
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/nix/weaviate.nix b/nix/weaviate.nix
index ebffe4ff..9dc311f3 100644
--- a/nix/weaviate.nix
+++ b/nix/weaviate.nix
@@ -31,11 +31,13 @@ in
       '';
     };
 
-    envs = lib.mkOption {
-      type = types.attrsOf (types.oneOf [ types.str types.int types.bool (types.listOf types.str) ]);
+    environment = lib.mkOption {
+      type = types.attrsOf (types.oneOf [ types.raw (types.listOf types.str) ]);
       default = { };
       description = ''
         Weaviate environment variables.
+
+        See https://weaviate.io/developers/weaviate/config-refs/env-vars
       '';
       example = lib.literalExpression ''
         {
@@ -56,32 +58,30 @@ in
         processes = {
           "${name}" =
             let
+              asAtom = value:
+                if builtins.isList value then lib.concatStringsSep "," value else value;
               toStr = value:
-                if builtins.isString value then builtins.toJSON value
-                else if builtins.isBool value then (if value then "true" else "false")
-                else if builtins.isList value then builtins.toJSON (lib.concatStringsSep "," value)
-                else if builtins.isInt value then toString value
-                else throw "Unrecognized type";
+                if builtins.isString value then value else builtins.toJSON value;
 
-              exports = (lib.mapAttrsToList (name: value: "export ${name}=${toStr value}") ({ "PERSISTENCE_DATA_PATH" = config.dataDir; }
-                // config.envs));
+              environment = (lib.mapAttrsToList (name: value: "${name}=${toStr (asAtom value)}") ({ "PERSISTENCE_DATA_PATH" = config.dataDir; }
+                // config.environment));
 
               startScript = pkgs.writeShellApplication {
                 name = "start-weaviate";
                 runtimeInputs = [ config.package ];
                 text = ''
-                  ${lib.concatStringsSep "\n" exports}
                   exec weaviate --scheme http --host ${config.host} --port ${toString config.port}
                 '';
               };
             in
             {
+              inherit environment;
+
               command = startScript;
 
               readiness_probe = {
                 http_get = {
-                  host = config.host;
-                  port = config.port;
+                  inherit (config) host port;
                   path = "/v1/.well-known/ready";
                 };
                 initial_delay_seconds = 3;
diff --git a/nix/weaviate_test.nix b/nix/weaviate_test.nix
index 5917b5b4..1753955d 100644
--- a/nix/weaviate_test.nix
+++ b/nix/weaviate_test.nix
@@ -1,7 +1,7 @@
 { pkgs, config, ... }: {
   services.weaviate."weaviate1" = {
     enable = true;
-    envs = {
+    environment = {
       # To fix the problem with nix failing to run weaviate in sandbox mode
       CLUSTER_ADVERTISE_ADDR = "127.0.0.1";
     };

From 3bdaf79ca8e9620974bb6ae697df0de813cdeabc Mon Sep 17 00:00:00 2001
From: Mahdi Seyedan <mahdi.se@yahoo.com>
Date: Mon, 27 May 2024 01:37:27 +0330
Subject: [PATCH 08/10] chore(weaviate): unnecessary null type for host option
 and better default path for dataDir

---
 nix/weaviate.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nix/weaviate.nix b/nix/weaviate.nix
index 9dc311f3..f10b4344 100644
--- a/nix/weaviate.nix
+++ b/nix/weaviate.nix
@@ -10,12 +10,12 @@ in
 
     dataDir = lib.mkOption {
       type = types.str;
-      default = "./data";
+      default = "./data/${name}";
       description = "Path to the Weaviate data store";
     };
 
     host = lib.mkOption {
-      type = types.nullOr types.str;
+      type = types.str;
       default = "127.0.0.1";
       description = ''
         The IP to listen on

From c2a6f5ee649b56e7f8f5ec61ce8e45da23065462 Mon Sep 17 00:00:00 2001
From: Mahdi Seyedan <mahdi.se@yahoo.com>
Date: Mon, 27 May 2024 01:53:32 +0330
Subject: [PATCH 09/10] refactor(weaviate): use environment options as attrset
 instead of list

---
 nix/weaviate.nix | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/nix/weaviate.nix b/nix/weaviate.nix
index f10b4344..e73909e8 100644
--- a/nix/weaviate.nix
+++ b/nix/weaviate.nix
@@ -1,6 +1,10 @@
 { pkgs, lib, name, config, ... }:
 let
   inherit (lib) types;
+  asAtom = value:
+    if builtins.isList value then lib.concatStringsSep "," value else value;
+  toStr = value:
+    if builtins.isString value then value else builtins.toJSON value;
 in
 {
   options = {
@@ -48,6 +52,8 @@ in
           ENABLE_MODULES = ["text2vec-openai" "generative-openai"];
         }
       '';
+      apply = attrs:
+        lib.mapAttrs (_: value: toStr (asAtom value)) attrs;
     };
 
     outputs.settings = lib.mkOption {
@@ -58,14 +64,6 @@ in
         processes = {
           "${name}" =
             let
-              asAtom = value:
-                if builtins.isList value then lib.concatStringsSep "," value else value;
-              toStr = value:
-                if builtins.isString value then value else builtins.toJSON value;
-
-              environment = (lib.mapAttrsToList (name: value: "${name}=${toStr (asAtom value)}") ({ "PERSISTENCE_DATA_PATH" = config.dataDir; }
-                // config.environment));
-
               startScript = pkgs.writeShellApplication {
                 name = "start-weaviate";
                 runtimeInputs = [ config.package ];
@@ -75,7 +73,7 @@ in
               };
             in
             {
-              inherit environment;
+              environment = config.environment // { "PERSISTENCE_DATA_PATH" = config.dataDir; };
 
               command = startScript;
 

From fc38330431114c82903a015654f7d6292e4955cd Mon Sep 17 00:00:00 2001
From: Mahdi Seyedan <mahdi.se@yahoo.com>
Date: Mon, 27 May 2024 03:29:23 +0330
Subject: [PATCH 10/10] refactor(weaviate): apply eta reduce to environment
 apply function

---
 nix/weaviate.nix | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/nix/weaviate.nix b/nix/weaviate.nix
index e73909e8..049c3ed2 100644
--- a/nix/weaviate.nix
+++ b/nix/weaviate.nix
@@ -52,8 +52,7 @@ in
           ENABLE_MODULES = ["text2vec-openai" "generative-openai"];
         }
       '';
-      apply = attrs:
-        lib.mapAttrs (_: value: toStr (asAtom value)) attrs;
+      apply = lib.mapAttrs (_: value: toStr (asAtom value));
     };
 
     outputs.settings = lib.mkOption {