fix last details

eclipsevortex · May 7, 2024 · 22ea24a · 22ea24a
1 parent ffc94e8
commit 22ea24a
Show file tree

Hide file tree

Showing 17 changed files with 424 additions and 102 deletions.
diff --git a/README.md b/README.md
@@ -376,6 +376,8 @@ pm2 start neurons/validator.py \
 
 > NOTE: to access the wandb UI to get statistics about the miners, you can click on this [link](https://wandb.ai/eclipsevortext/subvortex-team) and choose the validator run you want.
 
+> NOTE: by default the dumps created by the auto-update will be stored in /etc/redis. If you want to change the location, please use `--database.redis_dump_path`.
+
 ## Releases
 
 - [Release-2.2.0](./scripts/release/release-2.2.0/RELEASE-2.2.0.md)

diff --git a/neurons/validator.py b/neurons/validator.py
@@ -193,7 +193,8 @@ async def run(self):
         bt.logging.info("run()")
 
         # Initi versioin control
-        self.version_control = VersionControl(self.database)
+        dump_path = self.config.database.redis_dump_path
+        self.version_control = VersionControl(self.database, dump_path)
 
         # Init miners
         self.miners = await get_all_miners(self)
@@ -208,6 +209,17 @@ async def run(self):
 
         try:
             while 1:
+                # Start the upgrade process every 10 minutes
+                if should_upgrade(self.config.auto_update, self.last_upgrade_check):
+                    bt.logging.debug("Checking upgrade")
+                    must_restart = await self.version_control.upgrade()
+                    if must_restart:
+                        finish_wandb()
+                        self.version_control.restart()
+                        return
+
+                    self.last_upgrade_check = time.time()
+
                 start_epoch = time.time()
 
                 await resync_metagraph_and_miners(self)
@@ -266,17 +278,6 @@ async def run_forward():
                     prev_set_weights_block = get_current_block(self.subtensor)
                     save_state(self)
 
-                # Start the upgrade process every 10 minutes
-                if should_upgrade(self.config.auto_update, self.last_upgrade_check):
-                    bt.logging.debug("Checking upgrade")
-                    must_restart = await self.version_control.upgrade()
-                    if must_restart:
-                        finish_wandb()
-                        self.version_control.restart()
-                        return
-
-                    self.last_upgrade_check = time.time()
-
                 # Rollover wandb to a new run.
                 if should_reinit_wandb(self):
                     bt.logging.info("Reinitializing wandb")

diff --git a/scripts/redis/README.md b/scripts/redis/README.md
@@ -296,7 +296,7 @@ To create a Redis dump manually, you can use the python script `redis_dump.py`.
 For example, if you want to create the dump in the `subVortex` directory, you can run
 
 ```
-python3 ./scripts/redis/utils/redis_dump.py --run-type create
+python3 ./scripts/redis/utils/redis_dump.py --run-type create --dump-path redis-dump-2.0.0.json
 ```
 
 If you want to create the dump in another location and/or name, you can use the argument `--dump-path`
@@ -312,11 +312,11 @@ To restore a Redis dump manually, you can use the python script `redis_dump.py`.
 For example, if you want to create in `subVortex` directory, you can run
 
 ```
-python3 ./scripts/redis/utils/redis_dump.py --run-type restore
+python3 ./scripts/redis/utils/redis_dump.py --run-type restore --dump-path redis-dump-2.0.0.json
 ```
 
 If you want to restore a dump in another location, you can use the argument `--dump-path`
 
 ```
-python3 ./scripts/redis/utils/redis_dump.py --run-type restore --dump-path /tmp/redis
+python3 ./scripts/redis/utils/redis_dump.py --run-type restore --dump-path /tmp/redis/redis-dump-2.0.0.json
 ```
diff --git a/scripts/redis/utils/redis_dump.py b/scripts/redis/utils/redis_dump.py
@@ -71,7 +71,7 @@ async def main(args):
         parser.add_argument(
             "--dump-path",
             type=str,
-            default="",
+            default="/tmp/redis",
             help="Dump file (with path) to create or restore",
         )
         parser.add_argument(

diff --git a/scripts/release/release-2.2.3/RELEASE-2.2.3.md b/scripts/release/release-2.2.3/RELEASE-2.2.3.md
@@ -24,11 +24,13 @@ Previous Release: 2.2.2
 ## Rollout Process <a id="validator-rollout-process"></a>
 
 1. **Upgrade Subnet**: Fetch the remote tags
+
    ```bash
    git fetch --tags --force
    ```
 
    Then, checkout the new release tag
+
    ```bash
    git checkout tags/v2.2.3
    ```
@@ -40,13 +42,38 @@ Previous Release: 2.2.2
    pip install -e .
    ```
 
-2. **Restart validator**: Restart your validator to take the new version
+2. **Delete validator**: Remove your validator
 
    ```bash
-   pm2 restart validator-7
+   pm2 delete validator-7
    ```
 
-3. **Check logs**: Check the validator logs to see if you see some `New Block`
+   Use `pm2 show validator-7` to get the list of arguments you were using to be able to restore them in the step 3.
+
+3. **Start validator in auto-upgrade mode**: Start the validator by running in **Subvortex**
+
+   ```bash
+   pm2 start neurons/validator.py -f \
+      --name validator-7 \
+      --interpreter python3 -- \
+      --netuid 7 \
+      --wallet.name $WALLET_NAME \
+      --wallet.hotkey $HOTKEY_NAME \
+      --subtensor.chain_endpoint ws://$IP:9944 \
+      --logging.debug  \
+      --auto-update
+   ```
+
+   Replace **$WALLET_NAME**, **$HOTKEY_NAME** and **$IP** by the expected value.
+   If you had other arguments, please add them!
+
+   > IMPORTANT <br />
+   > Do not forget to provide the `--auto-update` argument.
+
+   > IMPORTANT <br />
+   > Use wandb without overriding the default value, as it will enable the Subvortex team to monitor the version of the validators and take action if necessary.
+
+4. **Check logs**: Check the validator logs to see if you see some `New Block`
    ```bash
    pm2 logs validator-7
    ```
@@ -58,6 +85,7 @@ Previous Release: 2.2.2
 If any issues arise during or after the rollout, follow these steps to perform a rollback:
 
 1. **Downgrade Subnet**: Checkout the previous release tag
+
    ```bash
    git checkout tags/v2.2.2
    ```
@@ -69,13 +97,34 @@ If any issues arise during or after the rollout, follow these steps to perform a
    pip install -e .
    ```
 
-2. **Restart validator**: Restart your validator to take the new version
+2. **Delete validator**: Remove your validator
 
    ```bash
-   pm2 restart validator-7
+   pm2 delete validator-7
    ```
 
-3. **Check logs**: Check the validator logs to see if you see some `New Block`
+   Use `pm2 show validator-7` to get the list of arguments you were using to be able to restore them in the step 3.
+
+3. **Start validator**: Start the validator by running in **Subvortex**
+
+   ```bash
+   pm2 start neurons/validator.py -f \
+      --name validator-7 \
+      --interpreter python3 -- \
+      --netuid 7 \
+      --wallet.name $WALLET_NAME \
+      --wallet.hotkey $HOTKEY_NAME \
+      --subtensor.chain_endpoint ws://$IP:9944 \
+      --logging.debug
+   ```
+
+   Replace **$WALLET_NAME**, **$HOTKEY_NAME** and **$IP** by the expected value.
+   If you had other arguments, please add them!
+
+   > IMPORTANT <br />
+   > Do not forget to remove the `--auto-update` argument.
+
+4. **Check logs**: Check the validator logs to see if you see some `New Block`
    ```bash
    pm2 logs validator-7
    ```
@@ -87,11 +136,13 @@ If any issues arise during or after the rollout, follow these steps to perform a
 ## Rollout Process <a id="miner-rollout-process"></a>
 
 1. **Upgrade Subnet**: Fetch the remote tags
+
    ```bash
    git fetch --tags --force
    ```
 
    Then, checkout the new release tag
+
    ```bash
    git checkout tags/v2.2.3
    ```
@@ -103,20 +154,42 @@ If any issues arise during or after the rollout, follow these steps to perform a
    pip install -e .
    ```
 
-2. **Restart validator**: Restart your validator to take the new version
+2. **Delete miner**: Remove your miner
 
    ```bash
-   pm2 restart miner-7
+   pm2 delete miner-7
    ```
 
-3. **Check logs**: Check the validator logs to see if you see some `New Block`
+   Use `pm2 show miner-7` to get the list of arguments you were using to be able to restore them in the step 3.
+
+3. **Start validator in auto-upgrade mode**: Start the miner by running in **Subvortex**
+
+   ```bash
+   pm2 start neurons/miner.py -f \
+      --name miner-7 \
+      --interpreter python3 -- \
+      --netuid 7 \
+      --wallet.name $WALLET_NAME \
+      --wallet.hotkey $HOTKEY_NAME \
+      --logging.debug  \
+      --auto-update
+   ```
+
+   Replace **$WALLET_NAME**, **$HOTKEY_NAME** and **$IP** by the expected value.
+   If you had other arguments, please add them!
+
+   > IMPORTANT <br />
+   > Do not forget to provide the `--auto-update` argument.
+
+4. **Check logs**: Check the miner logs to see if you see some `New Block`
    ```bash
    pm2 logs miner-7
    ```
 
 ## Rollback Process <a id="miner-rollback-process"></a>
 
 1. **Downgrade Subnet**: Checkout the previous release tag
+
    ```bash
    git checkout tags/v2.2.2
    ```
@@ -128,13 +201,33 @@ If any issues arise during or after the rollout, follow these steps to perform a
    pip install -e .
    ```
 
-2. **Restart validator**: Restart your validator to take the new version
+2. **Delete miner**: Remove your miner
+
+   ```bash
+   pm2 delete miner-7
+   ```
+
+   Use `pm2 show miner-7` to get the list of arguments you were using to be able to restore them in the step 3.
+
+3. **Start miner**: Start the miner by running in **Subvortex**
 
    ```bash
-   pm2 restart miner-7
+   pm2 start neurons/miner.py -f \
+      --name miner-7 \
+      --interpreter python3 -- \
+      --netuid 7 \
+      --wallet.name $WALLET_NAME \
+      --wallet.hotkey $HOTKEY_NAME \
+      --logging.debug
    ```
 
-3. **Check logs**: Check the validator logs to see if you see some `New Block`
+   Replace **$WALLET_NAME**, **$HOTKEY_NAME** and **$IP** by the expected value.
+   If you had other arguments, please add them!
+
+   > IMPORTANT <br />
+   > Do not forget to remove the `--auto-update` argument
+
+4. **Check logs**: Check the miner logs to see if you see some `New Block`
    ```bash
    pm2 logs miner-7
    ```

diff --git a/scripts/setup_and_run.sh b/scripts/setup_and_run.sh
@@ -212,7 +212,8 @@ if [[ "$TYPE" == "miner" ]]; then
     --subtensor.network local \
     --wallet.name $WALLET_NAME \
     --wallet.hotkey $HOTKEY_NAME \
-    --logging.debug
+    --logging.debug \
+    --auto-update
 fi
 
 # Run validator
@@ -239,6 +240,7 @@ if [[ "$TYPE" == "validator" ]]; then
     --netuid $NETUID \
     --wallet.name $WALLET_NAME \
     --wallet.hotkey $HOTKEY_NAME \
-    --logging.debug \
+    --logging.debug  \
+    --auto-update \
     $OPTIONS
 fi
diff --git a/scripts/subnet/README.md b/scripts/subnet/README.md
@@ -72,7 +72,7 @@ To uprade the Subnet manually, you can use the python script `subnet_upgrade.py`
 For example, if you are on tag v2.2.2 and want to migrate to the tag v2.2.3, you can run in `SubVortex`
 
 ```
-python3 ./scripts/subnet/subnet_upgrade.py --tag v2.2.3
+python3 ./scripts/subnet/utils/subnet_upgrade.py --tag v2.2.3
 ```
 
 ## Downgrade <a id="migration-downgrade"></a>
@@ -82,5 +82,5 @@ To downgrade the Subnet manually, you can use the python script `subnet_upgrade.
 For example, if you are on tag v2.2.3 and want to downgrade to the tag v2.2.2, you can run in `SubVortex`
 
 ```
-python3 ./scripts/subnet/subnet_upgrade.py --tag v2.2.2
+python3 ./scripts/subnet/utils/subnet_upgrade.py --tag v2.2.2
 ```
diff --git a/subnet/miner/run.py b/subnet/miner/run.py
@@ -7,6 +7,7 @@
 
 from subnet.miner.version import VersionControl
 
+
 def run(self):
     """
     Initiates and manages the main loop for the miner on the Bittensor network.
@@ -41,7 +42,7 @@ def run(self):
 
     netuid = self.config.netuid
 
-    version_control = VersionControl()
+    self.version_control = VersionControl()
 
     # Keep a track of last upgrade check
     self.last_upgrade_check = 0
@@ -63,11 +64,11 @@ def handler(obj, update_nr, subscription_id):
 
         if should_upgrade(self.config.auto_update, self.last_upgrade_check):
             bt.logging.debug("Checking upgrade")
-            must_restart = version_control.upgrade()
+            must_restart = self.version_control.upgrade()
             if must_restart:
                 self.version_control.restart()
                 return
-            
+
             self.last_upgrade_check = time.time()
 
         bt.logging.debug(
@@ -77,4 +78,4 @@ def handler(obj, update_nr, subscription_id):
         if self.should_exit:
             return True
 
-    block_handler_substrate.subscribe_block_headers(handler)
+    block_handler_substrate.subscribe_block_headers(handler)
diff --git a/subnet/shared/utils.py b/subnet/shared/utils.py
@@ -20,6 +20,10 @@
 import subprocess
 import bittensor as bt
 
+# Check if there is an update every 5 minutes
+# Github Rate Limit 60 requests per hour / per ip (Reference: https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28)
+CHECK_UPDATE_FREQUENCY = 5 * 60
+
 
 def get_redis_password(
     redis_password: str = None, redis_conf: str = "/etc/redis/redis.conf"
@@ -48,8 +52,7 @@ def get_redis_password(
 
 def should_upgrade(auto_update: bool, last_upgrade_check: float):
     """
-    True if it is sime to upgrade, false otherwise
-    For now, upgrading evering 60 seconds
+    True if it is time to upgrade, false otherwise
     """
     time_since_last_update = time.time() - last_upgrade_check
-    return time_since_last_update >= 60 and auto_update
+    return time_since_last_update >= CHECK_UPDATE_FREQUENCY and auto_update
diff --git a/subnet/validator/config.py b/subnet/validator/config.py
@@ -213,6 +213,12 @@ def add_args(cls, parser):
         help="Redis configuration path.",
         default="/etc/redis/redis.conf",
     )
+    parser.add_argument(
+        "--database.redis_dump_path",
+        type=str,
+        help="Redis directory where to store dumps.",
+        default="/etc/redis/",
+    )
 
     # Auto update
     parser.add_argument(