From e5bb57d7f0dbc305adfc7c8b332a4f9b27f2c878 Mon Sep 17 00:00:00 2001 From: Nico Arqueros <1622112+nicarq@users.noreply.github.com> Date: Fri, 13 Sep 2024 00:59:50 -0500 Subject: [PATCH 1/4] upgrade to 0.10.0 --- Cargo.lock | 681 ++++++------------ scripts/compile_bert_cpp.sh | 11 - .../curl_replacements/node1_create_message.sh | 11 - .../shinkai_create_message.sh | 11 - .../curl_replacements/sub1_create_message.sh | 16 - .../sub1_registration_code.sh | 16 - scripts/mistral_run_node1.sh | 29 - scripts/run_gpt_localhost.sh | 31 - scripts/run_identity_9652.sh | 29 - scripts/run_identity_9752.sh | 29 - scripts/run_mac_intel.sh | 30 - scripts/run_node1.sh | 38 - scripts/run_node1_shinkai_backend.sh | 31 - scripts/run_node1_together_ai.sh | 31 - scripts/run_node2.sh | 31 - scripts/run_requester_node.sh | 39 - scripts/run_streamer_tcp.sh | 30 - shinkai-bin/shinkai-node/Cargo.toml | 2 +- .../src/lance_db/shinkai_lance_db.rs | 10 +- .../src/lance_db/shinkai_lance_version.rs | 8 +- .../src/lance_db/shinkai_prompt_db.rs | 11 +- 21 files changed, 230 insertions(+), 895 deletions(-) delete mode 100644 scripts/compile_bert_cpp.sh delete mode 100755 scripts/curl_replacements/node1_create_message.sh delete mode 100755 scripts/curl_replacements/shinkai_create_message.sh delete mode 100755 scripts/curl_replacements/sub1_create_message.sh delete mode 100755 scripts/curl_replacements/sub1_registration_code.sh delete mode 100755 scripts/mistral_run_node1.sh delete mode 100755 scripts/run_gpt_localhost.sh delete mode 100755 scripts/run_identity_9652.sh delete mode 100755 scripts/run_identity_9752.sh delete mode 100755 scripts/run_mac_intel.sh delete mode 100755 scripts/run_node1.sh delete mode 100755 scripts/run_node1_shinkai_backend.sh delete mode 100755 scripts/run_node1_together_ai.sh delete mode 100755 scripts/run_node2.sh delete mode 100755 scripts/run_requester_node.sh delete mode 100755 scripts/run_streamer_tcp.sh diff --git a/Cargo.lock b/Cargo.lock index 116aab699..a0ccc2a41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -236,61 +236,25 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" -[[package]] -name = "arrow" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" -dependencies = [ - "arrow-arith 51.0.0", - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-csv 51.0.0", - "arrow-data 51.0.0", - "arrow-ipc 51.0.0", - "arrow-json 51.0.0", - "arrow-ord 51.0.0", - "arrow-row 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "arrow-string 51.0.0", -] - [[package]] name = "arrow" version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05048a8932648b63f21c37d88b552ccc8a65afb6dfe9fc9f30ce79174c2e7a85" dependencies = [ - "arrow-arith 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-cast 52.2.0", - "arrow-csv 52.2.0", - "arrow-data 52.2.0", - "arrow-ipc 52.2.0", - "arrow-json 52.2.0", - "arrow-ord 52.2.0", - "arrow-row 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", - "arrow-string 52.2.0", -] - -[[package]] -name = "arrow-arith" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "chrono", - "half", - "num", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] @@ -299,31 +263,15 @@ version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d8a57966e43bfe9a3277984a14c24ec617ad874e4c0e1d2a1b083a39cfbf22c" dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "half", "num", ] -[[package]] -name = "arrow-array" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" -dependencies = [ - "ahash 0.8.11", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "chrono", - "half", - "hashbrown 0.14.5", - "num", -] - [[package]] name = "arrow-array" version = "52.2.0" @@ -331,9 +279,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f4a9468c882dc66862cef4e1fd8423d47e67972377d85d80e022786427768c" dependencies = [ "ahash 0.8.11", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "chrono-tz 0.9.0", "half", @@ -341,17 +289,6 @@ dependencies = [ "num", ] -[[package]] -name = "arrow-buffer" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27" -dependencies = [ - "bytes", - "half", - "num", -] - [[package]] name = "arrow-buffer" version = "52.2.0" @@ -363,38 +300,17 @@ dependencies = [ "num", ] -[[package]] -name = "arrow-cast" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "atoi", - "base64 0.22.0", - "chrono", - "comfy-table", - "half", - "lexical-core", - "num", - "ryu", -] - [[package]] name = "arrow-cast" version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da26719e76b81d8bc3faad1d4dbdc1bcc10d14704e63dc17fc9f3e7e1e567c8e" dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "atoi", "base64 0.22.0", "chrono", @@ -405,36 +321,17 @@ dependencies = [ "ryu", ] -[[package]] -name = "arrow-csv" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "chrono", - "csv", - "csv-core", - "lazy_static", - "lexical-core", - "regex", -] - [[package]] name = "arrow-csv" version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c13c36dc5ddf8c128df19bab27898eea64bf9da2b555ec1cd17a8ff57fba9ec2" dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-cast 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono", "csv", "csv-core", @@ -443,91 +340,45 @@ dependencies = [ "regex", ] -[[package]] -name = "arrow-data" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" -dependencies = [ - "arrow-buffer 51.0.0", - "arrow-schema 51.0.0", - "half", - "num", -] - [[package]] name = "arrow-data" version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd9d6f18c65ef7a2573ab498c374d8ae364b4a4edf67105357491c031f716ca5" dependencies = [ - "arrow-buffer 52.2.0", - "arrow-schema 52.2.0", + "arrow-buffer", + "arrow-schema", "half", "num", ] -[[package]] -name = "arrow-ipc" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "flatbuffers 23.5.26", -] - [[package]] name = "arrow-ipc" version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e786e1cdd952205d9a8afc69397b317cfbb6e0095e445c69cda7e8da5c1eeb0f" dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-cast 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "flatbuffers 24.3.25", "lz4_flex", "zstd 0.13.2", ] -[[package]] -name = "arrow-json" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "chrono", - "half", - "indexmap 2.1.0", - "lexical-core", - "num", - "serde", - "serde_json", -] - [[package]] name = "arrow-json" version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb22284c5a2a01d73cebfd88a33511a3234ab45d66086b2ca2d1228c3498e445" dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-cast 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono", "half", "indexmap 2.1.0", @@ -537,51 +388,21 @@ dependencies = [ "serde_json", ] -[[package]] -name = "arrow-ord" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "half", - "num", -] - [[package]] name = "arrow-ord" version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42745f86b1ab99ef96d1c0bcf49180848a64fe2c7a7a0d945bc64fa2b21ba9bc" dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "half", "num", ] -[[package]] -name = "arrow-row" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" -dependencies = [ - "ahash 0.8.11", - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "half", - "hashbrown 0.14.5", -] - [[package]] name = "arrow-row" version = "52.2.0" @@ -589,19 +410,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd09a518c602a55bd406bcc291a967b284cfa7a63edfbf8b897ea4748aad23c" dependencies = [ "ahash 0.8.11", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", ] -[[package]] -name = "arrow-schema" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" - [[package]] name = "arrow-schema" version = "52.2.0" @@ -611,20 +426,6 @@ dependencies = [ "bitflags 2.4.0", ] -[[package]] -name = "arrow-select" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" -dependencies = [ - "ahash 0.8.11", - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "num", -] - [[package]] name = "arrow-select" version = "52.2.0" @@ -632,41 +433,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "600bae05d43483d216fb3494f8c32fdbefd8aa4e1de237e790dbb3d9f44690a3" dependencies = [ "ahash 0.8.11", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num", ] -[[package]] -name = "arrow-string" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "memchr", - "num", - "regex", - "regex-syntax 0.8.2", -] - [[package]] name = "arrow-string" version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0dc1985b67cb45f6606a248ac2b4a288849f196bab8c657ea5589f47cdd55e6" dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "memchr", "num", "regex", @@ -2711,10 +2495,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab9d55a9cd2634818953809f75ebe5248b00dd43c3227efb2a51a2d5feaad54e" dependencies = [ "ahash 0.8.11", - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-ipc 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-array", + "arrow-ipc", + "arrow-schema", "async-trait", "bytes", "chrono", @@ -2758,10 +2542,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "def66b642959e7f96f5d2da22e1f43d3bd35598f821e5ce351a0553e0f1b7367" dependencies = [ "ahash 0.8.11", - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", "chrono", "half", "hashbrown 0.14.5", @@ -2787,7 +2571,7 @@ version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ac0fd8b5d80bbca3fc3b6f40da4e9f6907354824ec3b18bbd83fee8cf5c3c3e" dependencies = [ - "arrow 52.2.0", + "arrow", "chrono", "dashmap", "datafusion-common", @@ -2809,9 +2593,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2103d2cc16fb11ef1fa993a6cac57ed5cb028601db4b97566c90e5fa77aa1e68" dependencies = [ "ahash 0.8.11", - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", + "arrow", + "arrow-array", + "arrow-buffer", "chrono", "datafusion-common", "paste", @@ -2827,7 +2611,7 @@ version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a369332afd0ef5bd565f6db2139fb9f1dfdd0afa75a7f70f000b74208d76994f" dependencies = [ - "arrow 52.2.0", + "arrow", "base64 0.22.0", "chrono", "datafusion-common", @@ -2850,8 +2634,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92718db1aff70c47e5abf9fc975768530097059e5db7c7b78cd64b5e9a11fc77" dependencies = [ "ahash 0.8.11", - "arrow 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -2867,11 +2651,11 @@ version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30bb80f46ff3dcf4bb4510209c2ba9b8ce1b716ac8b7bf70c6bf7dca6260c831" dependencies = [ - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-ord 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -2888,7 +2672,7 @@ version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82f34692011bec4fdd6fc18c264bf8037b8625d801e6dd8f5111af15cb6d71d3" dependencies = [ - "arrow 52.2.0", + "arrow", "async-trait", "chrono", "datafusion-common", @@ -2909,12 +2693,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45538630defedb553771434a437f7ca8f04b9b3e834344aafacecb27dc65d5e5" dependencies = [ "ahash 0.8.11", - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-ord 52.2.0", - "arrow-schema 52.2.0", - "arrow-string 52.2.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "arrow-string", "base64 0.22.0", "chrono", "datafusion-common", @@ -2939,7 +2723,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d8a72b0ca908e074aaeca52c14ddf5c28d22361e9cb6bc79bb733cd6661b536" dependencies = [ "ahash 0.8.11", - "arrow 52.2.0", + "arrow", "datafusion-common", "datafusion-expr", "hashbrown 0.14.5", @@ -2953,11 +2737,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b504eae6107a342775e22e323e9103f7f42db593ec6103b28605b7b7b1405c4a" dependencies = [ "ahash 0.8.11", - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-ord 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -2986,9 +2770,9 @@ version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5db33f323f41b95ae201318ba654a9bf11113e58a51a1dff977b1a836d3d889" dependencies = [ - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-array", + "arrow-schema", "datafusion-common", "datafusion-expr", "log 0.4.21", @@ -3936,16 +3720,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "flatbuffers" -version = "23.5.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" -dependencies = [ - "bitflags 1.3.2", - "rustc_version", -] - [[package]] name = "flatbuffers" version = "24.3.25" @@ -4033,11 +3807,10 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "0.1.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b5be8e6f9ffb7860aa8dd5998142374e4bfdf5c8c7f28fe963ce4b182829aab" +checksum = "26212c1db7eee3ec0808bd99107cf62ba4d3edd3489df601e2d0c73c5d739aec" dependencies = [ - "arrow 51.0.0", "rand 0.8.5", ] @@ -5225,18 +4998,18 @@ checksum = "3f35c735096c0293d313e8f2a641627472b83d01b937177fe76e5e2708d31e0d" [[package]] name = "lance" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d4c2e52b6ebad7e60e033ecca7f3369facdfb41823d4d3e87a1c32b84d6cfa" -dependencies = [ - "arrow 52.2.0", - "arrow-arith 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-ord 52.2.0", - "arrow-row 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a427160737dd74d2d4f566f3111027edc63927106541d173459d010209371c42" +dependencies = [ + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", "async-recursion", "async-trait", "async_cell", @@ -5256,7 +5029,6 @@ dependencies = [ "lance-arrow", "lance-core", "lance-datafusion", - "lance-datagen", "lance-encoding", "lance-file", "lance-index", @@ -5265,11 +5037,8 @@ dependencies = [ "lance-table", "lazy_static", "log 0.4.21", - "lru_time_cache", "moka", - "num_cpus", "object_store", - "ordered-float", "pin-project", "prost 0.12.6", "prost-build", @@ -5288,32 +5057,31 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18599d887554d3cb4c4b056850c452cc51f6788fecec6c3af2a4dffebbeea184" +checksum = "c2f1cfebe08c64b1edabe9b6ccd6f8ea1bc6349d0870d47f2db8cdadf02ab8e2" dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-cast 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", "getrandom 0.2.10", "half", "num-traits", "rand 0.8.5", - "serde", ] [[package]] name = "lance-core" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112becb10992ae62b5e5e7ea02e60302f9072afd0b4e3c45cecad58d6379879e" +checksum = "243aa2323dee6fcab6bb9bb3a21ae8f040c98a5de9bbfb7ab8484a036176185a" dependencies = [ - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-schema 52.2.0", + "arrow-array", + "arrow-buffer", + "arrow-schema", "async-trait", "byteorder", "bytes", @@ -5345,16 +5113,16 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cfb7663ed20daa3c9a910f971b7b521e37841046ae224aa032ab4c5e762fe7e" +checksum = "c0a69d039f93a43477245b51a8f1ce58a1f41485f8ded946f53031a11ded8c97" dependencies = [ - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-ord 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "arrow-select", "async-trait", "datafusion", "datafusion-common", @@ -5363,57 +5131,41 @@ dependencies = [ "futures", "lance-arrow", "lance-core", + "lazy_static", "log 0.4.21", "prost 0.12.6", "snafu", "tokio", ] -[[package]] -name = "lance-datagen" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afbfd57d950044b31b0d69c3e7637306aab56497f99668609b03304800292dea" -dependencies = [ - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-cast 52.2.0", - "arrow-schema 52.2.0", - "chrono", - "futures", - "hex", - "rand 0.8.5", - "rand_xoshiro", -] - [[package]] name = "lance-encoding" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a52cbffee5cf5263b98bfd9adccb10506aee5419e75f07980f5e6f298f112dca" -dependencies = [ - "arrow 52.2.0", - "arrow-arith 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-cast 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b713e49ce6039d0ca0f88e8ded66ee64d89c42f85107bc9e684fbff41386a65" +dependencies = [ + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", "bytes", "fsst", "futures", + "hex", "hyperloglogplus", + "itertools 0.12.1", "lance-arrow", "lance-core", - "lance-datagen", "log 0.4.21", "num-traits", - "num_cpus", "prost 0.12.6", "prost-build", "prost-types 0.12.6", "rand 0.8.5", - "rand_xoshiro", "snafu", "tokio", "tracing", @@ -5422,16 +5174,16 @@ dependencies = [ [[package]] name = "lance-file" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d4597be8c9380c63e1f164eca091804397fd2846a5f50bb2bf0cbf528a598f7" +checksum = "4e0eaebd40c77f8f06a0cbadcd07f9344aea616dcd4d8712f6cad81c2eda14d5" dependencies = [ - "arrow-arith 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "async-recursion", "async-trait", "byteorder", @@ -5441,13 +5193,10 @@ dependencies = [ "futures", "lance-arrow", "lance-core", - "lance-datagen", "lance-encoding", "lance-io", - "lance-testing", "log 0.4.21", "num-traits", - "num_cpus", "object_store", "prost 0.12.6", "prost-build", @@ -5461,15 +5210,15 @@ dependencies = [ [[package]] name = "lance-index" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7e6bfd777b30acbc80d7b5bf2f60739deaba752e2ec79f6ef07ba4eee50cbc7" +checksum = "d1ad5a42b9a4909749ee62fc94c64d19259c2aadca7c8446f42ee8e9c7a097d3" dependencies = [ - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-ord 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", + "arrow", + "arrow-array", + "arrow-ord", + "arrow-schema", + "arrow-select", "async-recursion", "async-trait", "bitvec", @@ -5494,15 +5243,14 @@ dependencies = [ "lance-table", "lazy_static", "log 0.4.21", + "moka", "num-traits", - "num_cpus", "object_store", "prost 0.12.6", "prost-build", "rand 0.8.5", "rayon", "roaring", - "rustc_version", "serde", "serde_json", "snafu", @@ -5510,22 +5258,23 @@ dependencies = [ "tempfile", "tokio", "tracing", + "uuid 1.8.0", ] [[package]] name = "lance-io" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c27b9892790fc463920c6092704d52f456f75ffa8ebe48d3c47aa9c253abbe" -dependencies = [ - "arrow 52.2.0", - "arrow-arith 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-cast 52.2.0", - "arrow-data 52.2.0", - "arrow-schema 52.2.0", - "arrow-select 52.2.0", +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0f334f2c279f80f19803141cf7f98c6b82e6ace3c7f75c8740f1df7a73bb720" +dependencies = [ + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", "async-priority-channel", "async-recursion", "async-trait", @@ -5539,7 +5288,7 @@ dependencies = [ "lance-arrow", "lance-core", "lazy_static", - "num_cpus", + "log 0.4.21", "object_store", "path_abs", "pin-project", @@ -5555,13 +5304,13 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a843ddcd2ffb1db81501fb432dbf2376ca01545ab48a26da7c8f33e4d79ece40" +checksum = "8fa019770a0afb287360a4ea919cff482371ad43318607d1e797534c819bf356" dependencies = [ - "arrow-array 52.2.0", - "arrow-ord 52.2.0", - "arrow-schema 52.2.0", + "arrow-array", + "arrow-ord", + "arrow-schema", "bitvec", "cc", "deepsize", @@ -5572,7 +5321,6 @@ dependencies = [ "lazy_static", "log 0.4.21", "num-traits", - "num_cpus", "rand 0.8.5", "rayon", "tokio", @@ -5581,15 +5329,15 @@ dependencies = [ [[package]] name = "lance-table" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "579a06930044e2b40cfe8b4f4d099e556875b484c79328dab1b8034e613e020c" +checksum = "99fa39bede133d578431db3f77b0c2c63fcaff12a088000648014c27266830a2" dependencies = [ - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-buffer 52.2.0", - "arrow-ipc 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ipc", + "arrow-schema", "async-trait", "aws-credential-types", "aws-sdk-dynamodb", @@ -5600,7 +5348,6 @@ dependencies = [ "futures", "lance-arrow", "lance-core", - "lance-datagen", "lance-file", "lance-io", "lazy_static", @@ -5623,12 +5370,12 @@ dependencies = [ [[package]] name = "lance-testing" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745542780246b10d9f6dbbd103029ada56d4d06933ed89faccc0b856df7da461" +checksum = "fb4eae0993cda6130cfd75d3d9d11830dd4ec8d4c66cf81def939837c419d4bc" dependencies = [ - "arrow-array 52.2.0", - "arrow-schema 52.2.0", + "arrow-array", + "arrow-schema", "lance-arrow", "num-traits", "rand 0.8.5", @@ -5636,17 +5383,17 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.8.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f379741ddacf238e142f10f9f8ce58c6d0009d4bd2bf86bd7a9b916fc38141f2" +checksum = "d867f115c1c23e77dc46a967664e3073630d53e3d61399f0b984c4a4753fd3a7" dependencies = [ - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-cast 52.2.0", - "arrow-data 52.2.0", - "arrow-ipc 52.2.0", - "arrow-ord 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-array", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", "async-trait", "bytes", "chrono", @@ -5655,8 +5402,10 @@ dependencies = [ "half", "lance", "lance-datafusion", + "lance-encoding", "lance-index", "lance-linalg", + "lance-table", "lance-testing", "lazy_static", "log 0.4.21", @@ -5962,12 +5711,6 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "lru_time_cache" -version = "0.11.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd" - [[package]] name = "lz4-sys" version = "1.9.4" @@ -6252,9 +5995,12 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa6e72583bf6830c956235bff0d5afec8cf2952f579ebad18ae7821a917d950f" dependencies = [ + "async-io", + "async-lock", "crossbeam-channel", "crossbeam-epoch", "crossbeam-utils", + "futures-util", "once_cell", "parking_lot 0.12.1", "quanta 0.11.1", @@ -7988,15 +7734,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core 0.6.4", -] - [[package]] name = "rangemap" version = "1.5.1" @@ -9212,9 +8949,9 @@ version = "0.8.3" dependencies = [ "aes-gcm", "anyhow", - "arrow 52.2.0", - "arrow-array 52.2.0", - "arrow-schema 52.2.0", + "arrow", + "arrow-array", + "arrow-schema", "async-channel", "async-lock", "async-std", diff --git a/scripts/compile_bert_cpp.sh b/scripts/compile_bert_cpp.sh deleted file mode 100644 index fc93cd630..000000000 --- a/scripts/compile_bert_cpp.sh +++ /dev/null @@ -1,11 +0,0 @@ -git clone https://github.com/robkorn/bert.cpp -cd bert.cpp -git submodule update --init --recursive -mkdir build -cd build -cmake .. -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -make -cd .. -cp build/bin/server ../bert-cpp-server -cd .. -rm -rf bert.cpp diff --git a/scripts/curl_replacements/node1_create_message.sh b/scripts/curl_replacements/node1_create_message.sh deleted file mode 100755 index de1661a51..000000000 --- a/scripts/curl_replacements/node1_create_message.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -export NODE_IP="127.0.0.1" -export NODE_PORT="8080" -export NODE_API_IP="127.0.0.1" -export NODE_API_PORT="3030" -export IDENTITY_SECRET_KEY="G2TyLP33XfqndppUzipoTWTs6XnKjmUhCQg1tH44isAG" -export ENCRYPTION_SECRET_KEY="FZ97ouxTGpNnmyyfSBxgC2FGHTpvo7mM7LWoMut6gEYx" -export PING_INTERVAL_SECS="0" - -cargo run -- --create_message --receiver_encryption_pk="8NT3CZR16VApT1B5zhinbAdqAvt8QkqMXEiojeFaGdgV" --recipient="@@node2.shinkai" diff --git a/scripts/curl_replacements/shinkai_create_message.sh b/scripts/curl_replacements/shinkai_create_message.sh deleted file mode 100755 index 0bdc6a002..000000000 --- a/scripts/curl_replacements/shinkai_create_message.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -export NODE_IP="127.0.0.1" -export NODE_PORT="8080" -export NODE_API_IP="127.0.0.1" -export NODE_API_PORT="3030" -export IDENTITY_SECRET_KEY="G2TyLP33XfqndppUzipoTWTs6XnKjmUhCQg1tH44isAG" -export ENCRYPTION_SECRET_KEY="FZ97ouxTGpNnmyyfSBxgC2FGHTpvo7mM7LWoMut6gEYx" -export PING_INTERVAL_SECS="0" - -./target/debug/shinkai_node --create_message --receiver_encryption_pk="8NT3CZR16VApT1B5zhinbAdqAvt8QkqMXEiojeFaGdgV" --recipient="@@node2.shinkai" diff --git a/scripts/curl_replacements/sub1_create_message.sh b/scripts/curl_replacements/sub1_create_message.sh deleted file mode 100755 index 991c5e93d..000000000 --- a/scripts/curl_replacements/sub1_create_message.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# sub1 identity_secret_key: 5NMac241NL3bwL1qtGQAPecup4ZHeCW5w47ehESi5gxv -# identity_public_key: AVjj3u4fL6V3gWqUoP7c3CjYrMH9gMthTpk7JjXKHUTw -# encryption_secret_key: 5NMac241NL3bwL1qtGQAPecup4ZHeCW5w47ehESi5gz2 -# encryption_public_key: 9eP1FY8k8BeVThK15vcLZLMV5t2KjbTMG6EqTMTpz7wR - -export NODE_IP="127.0.0.1" -export NODE_PORT="8084" -export NODE_API_IP="127.0.0.1" -export NODE_API_PORT="3033" -export IDENTITY_SECRET_KEY="5NMac241NL3bwL1qtGQAPecup4ZHeCW5w47ehESi5gxv" -export ENCRYPTION_SECRET_KEY="5NMac241NL3bwL1qtGQAPecup4ZHeCW5w47ehESi5gz2" -export PING_INTERVAL_SECS="0" - -cargo run -- --create_message --receiver_encryption_pk="8NT3CZR16VApT1B5zhinbAdqAvt8QkqMXEiojeFaGdgV" --recipient="@@node2.shinkai" diff --git a/scripts/curl_replacements/sub1_registration_code.sh b/scripts/curl_replacements/sub1_registration_code.sh deleted file mode 100755 index d61ed35b2..000000000 --- a/scripts/curl_replacements/sub1_registration_code.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# sub1 identity_secret_key: 5NMac241NL3bwL1qtGQAPecup4ZHeCW5w47ehESi5gxv -# identity_public_key: AVjj3u4fL6V3gWqUoP7c3CjYrMH9gMthTpk7JjXKHUTw -# encryption_secret_key: 5NMac241NL3bwL1qtGQAPecup4ZHeCW5w47ehESi5gz2 -# encryption_public_key: 9eP1FY8k8BeVThK15vcLZLMV5t2KjbTMG6EqTMTpz7wR - -export NODE_IP="127.0.0.1" -export NODE_PORT="8084" -export NODE_API_IP="127.0.0.1" -export NODE_API_PORT="3033" -export IDENTITY_SECRET_KEY="5NMac241NL3bwL1qtGQAPecup4ZHeCW5w47ehESi5gxv" -export ENCRYPTION_SECRET_KEY="5NMac241NL3bwL1qtGQAPecup4ZHeCW5w47ehESi5gz2" -export PING_INTERVAL_SECS="0" - -cargo run -- --create_message --receiver_encryption_pk="9BUoYQYq7K38mkk61q8aMH9kD9fKSVL1Fib7FbH6nUkQ" --recipient="@@node1.shinkai" --other="9BUoYQYq7K38mkk61q8aMH9kD9fKSVL1Fib7FbH6nUkQ" diff --git a/scripts/mistral_run_node1.sh b/scripts/mistral_run_node1.sh deleted file mode 100755 index add44997a..000000000 --- a/scripts/mistral_run_node1.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -export NODE_IP="0.0.0.0" -export NODE_PORT="9552" -export NODE_API_IP="0.0.0.0" -export NODE_API_PORT="9550" -export IDENTITY_SECRET_KEY="df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119" -export ENCRYPTION_SECRET_KEY="d83f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81159" -export PING_INTERVAL_SECS="0" -export GLOBAL_IDENTITY_NAME="@@localhost.arb-sep-shinkai" -export RUST_LOG=debug,error,info -export STARTING_NUM_QR_PROFILES="1" -export STARTING_NUM_QR_DEVICES="1" -export FIRST_DEVICE_NEEDS_REGISTRATION_CODE="false" -export LOG_SIMPLE="true" -export NO_SECRET_FILE="true" -export EMBEDDINGS_SERVER_URL="https://internal.shinkai.com/x-embed-api/" -export UNSTRUCTURED_SERVER_URL="https://internal.shinkai.com/x-unstructured-api/" - -export INITIAL_AGENT_NAMES="mistral" -export INITIAL_AGENT_URLS="https://api.together.xyz" -# export INITIAL_AGENT_MODELS="genericapi:mistralai/Mixtral-8x7B-Instruct-v0.1 -export INITIAL_AGENT_MODELS="genericapi:togethercomputer/llama-2-70b-chat" - - -# Add these lines to enable all log options -export LOG_ALL=1 - -cargo run diff --git a/scripts/run_gpt_localhost.sh b/scripts/run_gpt_localhost.sh deleted file mode 100755 index 2351b90bf..000000000 --- a/scripts/run_gpt_localhost.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -export NODE_IP="0.0.0.0" -export NODE_PORT="9852" -export NODE_API_IP="0.0.0.0" -export NODE_API_PORT="9850" -export NODE_WS_PORT="9851" -export IDENTITY_SECRET_KEY="df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119" -export ENCRYPTION_SECRET_KEY="d83f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81159" -export PING_INTERVAL_SECS="0" -export GLOBAL_IDENTITY_NAME="@@localhost.arb-sep-shinkai" -export NODE_STORAGE_PATH="storage_streamer" -export RUST_LOG=debug,error,info -export STARTING_NUM_QR_PROFILES="1" -export STARTING_NUM_QR_DEVICES="1" -export FIRST_DEVICE_NEEDS_REGISTRATION_CODE="false" -export LOG_SIMPLE="true" -export NO_SECRET_FILE="true" -export EMBEDDINGS_SERVER_URL="https://public.shinkai.com/x-em" -export UNSTRUCTURED_SERVER_URL="https://public.shinkai.com/x-un" - -export INITIAL_AGENT_NAMES="my_gpt,my_gpt_vision" -export INITIAL_AGENT_URLS="https://api.openai.com,https://api.openai.com" -export INITIAL_AGENT_MODELS="openai:gpt-4-1106-preview,openai:gpt-4-vision-preview" - -export RPC_URL="https://rpc.sepolia.org" -export CONTRACT_ADDRESS="0x1d2D57F78Bc3B878aF68c411a03AcF327c85e0D6" -# Add these lines to enable all log options -export LOG_ALL=1 - -cargo run diff --git a/scripts/run_identity_9652.sh b/scripts/run_identity_9652.sh deleted file mode 100755 index c349c4dc5..000000000 --- a/scripts/run_identity_9652.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -export NODE_IP="0.0.0.0" -export NODE_PORT="9652" -export NODE_WS_PORT="9651" -export NODE_API_IP="0.0.0.0" -export NODE_API_PORT="9650" -export NODE_STORAGE_PATH="storage_9652" -export IDENTITY_SECRET_KEY="bf0be9c2da2d5f9371bb61f5b2b9f4a6bb294f064e187056005a8bda8dc2ef00" -export ENCRYPTION_SECRET_KEY="806bbcca2ab460aaa57ed00aa2fdf88b1d039b5ca2d89306d4ebd77b14e52c77" -export PING_INTERVAL_SECS="0" -export GLOBAL_IDENTITY_NAME="@@_my_9652.arb-sep-shinkai" -export RUST_LOG=debug,error,info -export STARTING_NUM_QR_PROFILES="1" -export STARTING_NUM_QR_DEVICES="1" -export FIRST_DEVICE_NEEDS_REGISTRATION_CODE="false" -export LOG_SIMPLE="true" -export NO_SECRET_FILE="true" -export EMBEDDINGS_SERVER_URL="https://internal.shinkai.com/x-embed-api/" -export UNSTRUCTURED_SERVER_URL="https://internal.shinkai.com/x-unstructured-api/" - -export INITIAL_AGENT_NAMES="my_gpt,my_gpt_vision" -export INITIAL_AGENT_URLS="https://api.openai.com,https://api.openai.com" -export INITIAL_AGENT_MODELS="openai:gpt-4-1106-preview,openai:gpt-4-vision-preview" - -# Add these lines to enable all log options -export LOG_ALL=1 - -cargo run diff --git a/scripts/run_identity_9752.sh b/scripts/run_identity_9752.sh deleted file mode 100755 index fdf61829b..000000000 --- a/scripts/run_identity_9752.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -export NODE_IP="0.0.0.0" -export NODE_PORT="9752" -export NODE_WS_PORT="9751" -export NODE_API_IP="0.0.0.0" -export NODE_API_PORT="9750" -export NODE_STORAGE_PATH="storage_9750" -export IDENTITY_SECRET_KEY="fd1ca428ec1be6ae8b0b3d23ea507eba8cf7da0869578753b9781efda2b6a8ab" -export ENCRYPTION_SECRET_KEY="e06a1c02d638d4552d733dca8ff8f023841d1126965050b2048f1140bfd82a5c" -export PING_INTERVAL_SECS="0" -export GLOBAL_IDENTITY_NAME="@@_my_9752.arb-sep-shinkai" -export RUST_LOG=debug,error,info -export STARTING_NUM_QR_PROFILES="1" -export STARTING_NUM_QR_DEVICES="1" -export FIRST_DEVICE_NEEDS_REGISTRATION_CODE="false" -export LOG_SIMPLE="true" -export NO_SECRET_FILE="true" -export EMBEDDINGS_SERVER_URL="https://internal.shinkai.com/x-embed-api/" -export UNSTRUCTURED_SERVER_URL="https://internal.shinkai.com/x-unstructured-api/" - -export INITIAL_AGENT_NAMES="my_gpt,my_gpt_vision" -export INITIAL_AGENT_URLS="https://api.openai.com,https://api.openai.com" -export INITIAL_AGENT_MODELS="openai:gpt-4-1106-preview,openai:gpt-4-vision-preview" - -# Add these lines to enable all log options -export LOG_ALL=1 - -cargo run diff --git a/scripts/run_mac_intel.sh b/scripts/run_mac_intel.sh deleted file mode 100755 index 456e65e42..000000000 --- a/scripts/run_mac_intel.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -export NODE_IP="0.0.0.0" -export NODE_PORT="9552" -export NODE_API_IP="0.0.0.0" -export NODE_API_PORT="9550" -export IDENTITY_SECRET_KEY="df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119" -export ENCRYPTION_SECRET_KEY="d83f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81159" -export PING_INTERVAL_SECS="0" -export GLOBAL_IDENTITY_NAME="@@localhost.arb-sep-shinkai" -export RUST_LOG=debug,error,info -export STARTING_NUM_QR_PROFILES="1" -export STARTING_NUM_QR_DEVICES="1" -export FIRST_DEVICE_NEEDS_REGISTRATION_CODE="false" -export LOG_SIMPLE="true" -export NO_SECRET_FILE="true" -export EMBEDDINGS_SERVER_URL="https://internal.shinkai.com/x-embed-api/" -export UNSTRUCTURED_SERVER_URL="https://internal.shinkai.com/x-unstructured-api/" - -export INITIAL_AGENT_NAMES="o_mixtral,o_llava" -export INITIAL_AGENT_URLS="http://localhost:11435,http://localhost:11435" -export INITIAL_AGENT_MODELS="ollama:mixtral:8x7b-instruct-v0.1-q4_1,ollama:llava" -export INITIAL_AGENT_API_KEYS="," - -# Add these lines to enable all log options -export LOG_ALL=1 - -mkdir -p ./db/ - -./target/x86_64-apple-darwin/debug/shinkai_node diff --git a/scripts/run_node1.sh b/scripts/run_node1.sh deleted file mode 100755 index 58cb57fb6..000000000 --- a/scripts/run_node1.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -export NODE_IP="0.0.0.0" -export NODE_PORT="9452" -export NODE_API_IP="0.0.0.0" -export NODE_API_PORT="9450" -export NODE_WS_PORT="9451" -export IDENTITY_SECRET_KEY="df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119" -export ENCRYPTION_SECRET_KEY="d83f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81159" -export PING_INTERVAL_SECS="0" -# export GLOBAL_IDENTITY_NAME="@@localhost.arb-sep-shinkai" -export RUST_LOG=debug,error,info -export STARTING_NUM_QR_PROFILES="1" -export STARTING_NUM_QR_DEVICES="1" -export FIRST_DEVICE_NEEDS_REGISTRATION_CODE="false" -export LOG_SIMPLE="true" -export NO_SECRET_FILE="true" -export EMBEDDINGS_SERVER_URL="http://localhost:11434" -export UNSTRUCTURED_SERVER_URL="https://public.shinkai.com/x-un" - -# export TELEMETRY_ENDPOINT="https://apm-node-b1.shinkai.com/api/default" -# export TELEMETRY_AUTH_HEADER="Basic xxx" - -export STATIC_SERVER_PORT="9554" -export STATIC_SERVER_IP="0.0.0.0" -export STATIC_SERVER_FOLDER="./static_server_example" - -export INITIAL_AGENT_NAMES="my_gpt,my_gpt_vision,groq,llama3_gradient,llama3_8b,llama_3_together" -export INITIAL_AGENT_URLS="https://api.openai.com,https://api.openai.com,https://api.groq.com/openai/v1,http://localhost:11434,http://localhost:11434,https://api.together.xyz" -export INITIAL_AGENT_MODELS="openai:gpt-4-1106-preview,openai:gpt-4-vision-preview,groq:llama3-8b-8192,ollama:llama3-gradient:8b-instruct-1048k-q3_K_M,ollama:llama3:8b-instruct-q4_1,genericapi:meta-llama/Llama-3-8b-chat-hf" -export PROXY_IDENTITY="@@relayer_pub_01.arb-sep-shinkai" - -export CONTRACT_ADDRESS="0x1d2D57F78Bc3B878aF68c411a03AcF327c85e0D6" - -# Add these lines to enable all log options -export LOG_ALL=1 - -cargo run diff --git a/scripts/run_node1_shinkai_backend.sh b/scripts/run_node1_shinkai_backend.sh deleted file mode 100755 index 43e37cbc2..000000000 --- a/scripts/run_node1_shinkai_backend.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -export NODE_IP="0.0.0.0" -export NODE_PORT="9552" -export NODE_API_IP="0.0.0.0" -export NODE_API_PORT="9550" -export IDENTITY_SECRET_KEY="df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119" -export ENCRYPTION_SECRET_KEY="d83f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81159" -export PING_INTERVAL_SECS="0" -export GLOBAL_IDENTITY_NAME="@@localhost.arb-sep-shinkai" -export RUST_LOG=debug,error,info -export STARTING_NUM_QR_PROFILES="1" -export STARTING_NUM_QR_DEVICES="1" -export FIRST_DEVICE_NEEDS_REGISTRATION_CODE="false" -export LOG_SIMPLE="true" -export NO_SECRET_FILE="true" -export EMBEDDINGS_SERVER_URL="https://internal.shinkai.com/x-embed-api/" -export UNSTRUCTURED_SERVER_URL="https://internal.shinkai.com/x-unstructured-api/" - -export STATIC_SERVER_PORT="9554" -export STATIC_SERVER_IP="0.0.0.0" -export STATIC_SERVER_FOLDER="./static_server_example" - -export INITIAL_AGENT_NAMES="premium_gpt,standard_mixtral,gpt_vision" -export INITIAL_AGENT_URLS="https://dev-backend-hosting.shinkai.com,https://dev-backend-hosting.shinkai.com,https://dev-backend-hosting.shinkai.com" -export INITIAL_AGENT_MODELS="shinkai-backend:PREMIUM_TEXT_INFERENCE,shinkai-backend:STANDARD_TEXT_INFERENCE,shinkai-backend:PREMIUM_VISION_INFERENCE" - -# Add these lines to enable all log options -export LOG_ALL=1 - -cargo run diff --git a/scripts/run_node1_together_ai.sh b/scripts/run_node1_together_ai.sh deleted file mode 100755 index 24bce7c9f..000000000 --- a/scripts/run_node1_together_ai.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -export NODE_IP="0.0.0.0" -export NODE_PORT="9552" -export NODE_API_IP="0.0.0.0" -export NODE_API_PORT="9550" -export IDENTITY_SECRET_KEY="df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119" -export ENCRYPTION_SECRET_KEY="d83f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81159" -export PING_INTERVAL_SECS="0" -export GLOBAL_IDENTITY_NAME="@@localhost.arb-sep-shinkai" -export RUST_LOG=debug,error,info -export STARTING_NUM_QR_PROFILES="1" -export STARTING_NUM_QR_DEVICES="1" -export FIRST_DEVICE_NEEDS_REGISTRATION_CODE="false" -export LOG_SIMPLE="true" -export NO_SECRET_FILE="true" -export EMBEDDINGS_SERVER_URL="https://internal.shinkai.com/x-embed-api/" -export UNSTRUCTURED_SERVER_URL="https://internal.shinkai.com/x-unstructured-api/" - -export STATIC_SERVER_PORT="9554" -export STATIC_SERVER_IP="0.0.0.0" -export STATIC_SERVER_FOLDER="./static_server_example" - -export INITIAL_AGENT_NAMES="premium_gpt,standard_mixtral,gpt_vision,llama_3_together" -export INITIAL_AGENT_URLS="https://dev-backend-hosting.shinkai.com,https://dev-backend-hosting.shinkai.com,https://dev-backend-hosting.shinkai.com,https://api.together.xyz" -export INITIAL_AGENT_MODELS="shinkai-backend:PREMIUM_TEXT_INFERENCE,shinkai-backend:STANDARD_TEXT_INFERENCE,shinkai-backend:PREMIUM_VISION_INFERENCE,genericapi:meta-llama/Llama-3-8b-chat-hf" - -# Add these lines to enable all log options -export LOG_ALL=1 - -cargo run diff --git a/scripts/run_node2.sh b/scripts/run_node2.sh deleted file mode 100755 index c68639784..000000000 --- a/scripts/run_node2.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -export NODE_IP="127.0.0.1" -export NODE_PORT="8081" -export NODE_API_IP="0.0.0.0" -export NODE_API_PORT="8089" -export IDENTITY_SECRET_KEY="67abdd721024f0ff4e0b3f4c2fc13bc5bad42d0b7851d456d88d203d15aaa450" -export ENCRYPTION_SECRET_KEY="60abdd721024f0ff4e0b3f4c2fc13bc5bad42d0b7851d456d88d203d15aaa450" -export PING_INTERVAL_SECS="0" -export GLOBAL_IDENTITY_NAME="@@node2.shinkai" -export RUST_LOG=warn,error,info,debug -export EMBEDDINGS_SERVER_URL="https://internal.shinkai.com/x-embed-api/" -export UNSTRUCTURED_SERVER_URL="https://internal.shinkai.com/x-unstructured-api/" - -if [ -z "$1" ] -then - echo "No argument supplied for CONNECT_PK, using empty string" - export CONNECT_PK="" -else - export CONNECT_PK=$1 -fi - -if [ -z "$2" ] -then - echo "No argument supplied for CONNECT_ADDR, using empty string" - export CONNECT_ADDR="" -else - export CONNECT_ADDR=$2 -fi - -cargo run diff --git a/scripts/run_requester_node.sh b/scripts/run_requester_node.sh deleted file mode 100755 index b1ea6e601..000000000 --- a/scripts/run_requester_node.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -export NODE_IP="0.0.0.0" -export NODE_PORT="9952" -export NODE_API_IP="0.0.0.0" -export NODE_API_PORT="9950" -export NODE_WS_PORT="9951" -export IDENTITY_SECRET_KEY="df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119" -export ENCRYPTION_SECRET_KEY="d83f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81159" -export PING_INTERVAL_SECS="0" -export GLOBAL_IDENTITY_NAME="@@nico_requester.arb-sep-shinkai" -export NODE_STORAGE_PATH="storage_requester" -export RUST_LOG=debug,error,info -export STARTING_NUM_QR_PROFILES="1" -export STARTING_NUM_QR_DEVICES="1" -export FIRST_DEVICE_NEEDS_REGISTRATION_CODE="false" -export LOG_SIMPLE="true" -export NO_SECRET_FILE="true" -export EMBEDDINGS_SERVER_URL="https://public.shinkai.com/x-em" -export UNSTRUCTURED_SERVER_URL="https://public.shinkai.com/x-un" - -# export TELEMETRY_ENDPOINT="https://apm-node-b1.shinkai.com/api/default" -# export TELEMETRY_AUTH_HEADER="Basic xxx" - -export STATIC_SERVER_PORT="9954" -export STATIC_SERVER_IP="0.0.0.0" -export STATIC_SERVER_FOLDER="./static_server_example" - -export INITIAL_AGENT_NAMES="my_gpt,my_gpt_vision" -export INITIAL_AGENT_URLS="https://api.openai.com,https://api.openai.com" -export INITIAL_AGENT_MODELS="openai:gpt-4-1106-preview,openai:gpt-4-vision-preview" - -export RPC_URL="https://rpc.sepolia.org" -export CONTRACT_ADDRESS="0x1d2D57F78Bc3B878aF68c411a03AcF327c85e0D6" - -# Add these lines to enable all log options -export LOG_ALL=1 -export RUSTFLAGS="--cfg tokio_unstable" -cargo run --features console diff --git a/scripts/run_streamer_tcp.sh b/scripts/run_streamer_tcp.sh deleted file mode 100755 index 05158f325..000000000 --- a/scripts/run_streamer_tcp.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -export NODE_IP="0.0.0.0" -export NODE_PORT="9852" -export NODE_API_IP="0.0.0.0" -export NODE_API_PORT="9850" -export NODE_WS_PORT="9851" -export IDENTITY_SECRET_KEY="df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119" -export ENCRYPTION_SECRET_KEY="d83f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81159" -export PING_INTERVAL_SECS="0" -export GLOBAL_IDENTITY_NAME="@@external_identity_testing_tcp_relay.arb-sep-shinkai" -export NODE_STORAGE_PATH="storage_streamer_tcp" -export RUST_LOG=debug,error,info -export STARTING_NUM_QR_PROFILES="1" -export STARTING_NUM_QR_DEVICES="1" -export FIRST_DEVICE_NEEDS_REGISTRATION_CODE="false" -export LOG_SIMPLE="true" -export NO_SECRET_FILE="true" -export EMBEDDINGS_SERVER_URL="http://localhost:11434/" -export UNSTRUCTURED_SERVER_URL="https://public.shinkai.com/x-un" - -export INITIAL_AGENT_NAMES="my_gpt,my_gpt_vision" -export INITIAL_AGENT_URLS="https://api.openai.com,https://api.openai.com" -export INITIAL_AGENT_MODELS="openai:gpt-4-1106-preview,openai:gpt-4-vision-preview" - -export CONTRACT_ADDRESS="0x1d2D57F78Bc3B878aF68c411a03AcF327c85e0D6" -# Add these lines to enable all log options -export LOG_ALL=1 - -cargo run diff --git a/shinkai-bin/shinkai-node/Cargo.toml b/shinkai-bin/shinkai-node/Cargo.toml index ee71be8e0..0ade017ca 100644 --- a/shinkai-bin/shinkai-node/Cargo.toml +++ b/shinkai-bin/shinkai-node/Cargo.toml @@ -78,7 +78,7 @@ rust_decimal = "1.17.0" aws-types = "1.2.0" aws-config = { version = "1.2.1", features = ["behavior-version-latest"] } shinkai_tools_runner = { version = "0.7.13", features = ["built-in-tools"] } -lancedb = "0.8.0" +lancedb = "0.10.0" arrow = "52.1" arrow-array = "52.1" arrow-schema = "52.1" diff --git a/shinkai-bin/shinkai-node/src/lance_db/shinkai_lance_db.rs b/shinkai-bin/shinkai-node/src/lance_db/shinkai_lance_db.rs index 6809daac8..a86f92267 100644 --- a/shinkai-bin/shinkai-node/src/lance_db/shinkai_lance_db.rs +++ b/shinkai-bin/shinkai-node/src/lance_db/shinkai_lance_db.rs @@ -5,6 +5,7 @@ use arrow_array::{Array, BinaryArray, BooleanArray}; use arrow_array::{FixedSizeListArray, Float32Array, RecordBatch, RecordBatchIterator, StringArray}; use arrow_schema::{DataType, Field}; use futures::{StreamExt, TryStreamExt}; +use lancedb::connection::LanceFileVersion; use lancedb::index::Index; use lancedb::query::QueryBase; use lancedb::query::{ExecutableQuery, Select}; @@ -22,7 +23,7 @@ use super::shinkai_lancedb_error::ShinkaiLanceDBError; use super::shinkai_tool_schema::ShinkaiToolSchema; // Note: Add 1 to the current number to force an old fashion migration (delete all and then add all) -pub static LATEST_ROUTER_DB_VERSION: &str = "4"; +pub static LATEST_ROUTER_DB_VERSION: &str = "5"; // TODO: we need a way to export and import the db (or tables). it could be much faster to reset. @@ -75,8 +76,9 @@ impl LanceShinkaiDb { .map_err(|e| ShinkaiLanceDBError::Schema(e.to_string()))?; let table = match connection - .create_empty_table("tool_router_v4", schema) - // .data_storage_version(LanceFileVersion::V2_1) + .create_empty_table("tool_router_v5", schema) + .data_storage_version(LanceFileVersion::V2_1) + .enable_v2_manifest_paths(true) .execute() .await { @@ -85,7 +87,7 @@ impl LanceShinkaiDb { if let LanceDbError::TableAlreadyExists { .. } = e { // If the table already exists, retrieve and return it connection - .open_table("tool_router_v4") + .open_table("tool_router_v5") .execute() .await .map_err(ShinkaiLanceDBError::from)? diff --git a/shinkai-bin/shinkai-node/src/lance_db/shinkai_lance_version.rs b/shinkai-bin/shinkai-node/src/lance_db/shinkai_lance_version.rs index 775454867..dcb3035f4 100644 --- a/shinkai-bin/shinkai-node/src/lance_db/shinkai_lance_version.rs +++ b/shinkai-bin/shinkai-node/src/lance_db/shinkai_lance_version.rs @@ -4,6 +4,7 @@ use arrow_array::Array; use arrow_array::{RecordBatch, RecordBatchIterator, StringArray}; use arrow_schema::{DataType, Field}; use futures::TryStreamExt; +use lancedb::connection::LanceFileVersion; use lancedb::query::ExecutableQuery; use lancedb::query::QueryBase; use lancedb::Table; @@ -15,14 +16,15 @@ impl LanceShinkaiDb { let schema = arrow_schema::Schema::new(vec![Field::new("version", DataType::Utf8, false)]); match connection - .create_empty_table("version", schema.into()) - // .data_storage_version(LanceFileVersion::V2_1) + .create_empty_table("version_v2", schema.into()) + .data_storage_version(LanceFileVersion::V2_1) + .enable_v2_manifest_paths(true) .execute() .await { Ok(table) => Ok(table), Err(LanceDbError::TableAlreadyExists { .. }) => connection - .open_table("version") + .open_table("version_v2") .execute() .await .map_err(ShinkaiLanceDBError::from), diff --git a/shinkai-bin/shinkai-node/src/lance_db/shinkai_prompt_db.rs b/shinkai-bin/shinkai-node/src/lance_db/shinkai_prompt_db.rs index 59d2e63b5..27f9a7ffa 100644 --- a/shinkai-bin/shinkai-node/src/lance_db/shinkai_prompt_db.rs +++ b/shinkai-bin/shinkai-node/src/lance_db/shinkai_prompt_db.rs @@ -6,6 +6,7 @@ use arrow_array::{Array, FixedSizeListArray, Float32Array}; use arrow_array::{BooleanArray, RecordBatch, RecordBatchIterator, StringArray}; use arrow_schema::{DataType, Field}; use futures::{StreamExt, TryStreamExt}; +use lancedb::connection::LanceFileVersion; use lancedb::query::{ExecutableQuery, QueryBase, Select}; use lancedb::{table::AddDataMode, Connection, Error as LanceDbError, Table}; use shinkai_vector_resources::model_type::EmbeddingModelType; @@ -18,10 +19,16 @@ impl LanceShinkaiDb { ) -> Result