From c52c30124d42a989b4f7e35ed38c4f84a7db7c0c Mon Sep 17 00:00:00 2001 From: eclipsevortex Date: Fri, 3 May 2024 13:40:59 +0100 Subject: [PATCH 1/5] Add unit tests for resync miners --- subnet/validator/config.py | 2 +- subnet/validator/miner.py | 8 +- subnet/validator/models.py | 26 ++ tests/unit_tests/mocks/mock_country.py | 6 + tests/unit_tests/mocks/mock_redis.py | 13 + tests/unit_tests/subnet/test_get_next_uids.py | 4 +- .../subnet/test_get_uids_selection.py | 2 +- .../test_get_available_query_miners.py | 2 +- .../validator/test_get_available_uids.py | 2 +- .../subnet/validator/test_resync_miners.py | 294 ++++++++++++++++++ tests/unit_tests/utils/__init__.py | 0 tests/unit_tests/utils/metagraph.py | 77 +++++ tests/unit_tests/{ => utils}/utils.py | 0 13 files changed, 427 insertions(+), 9 deletions(-) create mode 100644 tests/unit_tests/mocks/mock_country.py create mode 100644 tests/unit_tests/subnet/validator/test_resync_miners.py create mode 100644 tests/unit_tests/utils/__init__.py create mode 100644 tests/unit_tests/utils/metagraph.py rename tests/unit_tests/{ => utils}/utils.py (100%) diff --git a/subnet/validator/config.py b/subnet/validator/config.py index 234ed994..7f221de2 100644 --- a/subnet/validator/config.py +++ b/subnet/validator/config.py @@ -114,7 +114,7 @@ def check_config(cls, config: "bt.Config"): def add_args(cls, parser): # Netuid Arg - parser.add_argument("--netuid", type=int, help="Storage network netuid", default=21) + parser.add_argument("--netuid", type=int, help="Subvortex network netuid", default=7) parser.add_argument( "--neuron.name", diff --git a/subnet/validator/miner.py b/subnet/validator/miner.py index 1a2acd8c..bf1dd18f 100644 --- a/subnet/validator/miner.py +++ b/subnet/validator/miner.py @@ -39,7 +39,7 @@ async def get_all_miners(self) -> List[Miner]: # Get all the ips from available miners ips = [self.metagraph.axons[uid].ip for uid in uids] - + for uid in uids: axon = self.metagraph.axons[uid] @@ -147,7 +147,7 @@ async def remove_miner(self, uid: int, hotkey: str): miners = [miner for miner in self.miners if miner.uid != uid] if len(miners) >= len(self.miners): return False - + # Remove the statistics await remove_hotkey_stastitics(hotkey, self.database) @@ -176,7 +176,9 @@ async def resync_miners(self): if not is_available: removed = await remove_miner(self, uid, hotkey) if removed: - bt.logging.success(f"[{uid}] Miner {hotkey} has been removed from the list") + bt.logging.success( + f"[{uid}] Miner {hotkey} has been removed from the list" + ) continue miner: Miner = next((miner for miner in self.miners if miner.uid == uid), None) diff --git a/subnet/validator/models.py b/subnet/validator/models.py index 984cddb1..b457e1d7 100644 --- a/subnet/validator/models.py +++ b/subnet/validator/models.py @@ -103,3 +103,29 @@ def __str__(self): def __repr__(self): return f"Miner(uid={self.uid}, hotkey={self.hotkey}, ip={self.ip}, ip_occurences={self.ip_occurences}, version={self.version}, country={self.country}, verified={self.verified}, sync={self.sync}, suspicious={self.suspicious}, score={self.score}, availability_score={self.availability_score}, latency_score={self.latency_score}, reliability_score={self.reliability_score}, distribution_score={self.distribution_score}, challenge_attempts={self.challenge_attempts}, challenge_successes={self.challenge_successes}, process_time={self.process_time})" + + def __eq__(self, other): + if isinstance(other, Miner): + return ( + self.uid == other.uid and + self.hotkey == other.hotkey and + self.ip == other.ip and + self.ip_occurences == other.ip_occurences and + self.version == other.version and + self.country == other.country and + self.score == other.score and + self.availability_score == other.availability_score and + self.reliability_score == other.reliability_score and + self.latency_score == other.latency_score and + self.distribution_score == other.distribution_score and + self.challenge_attempts == other.challenge_attempts and + self.challenge_successes == other.challenge_successes and + self.process_time == other.process_time and + self.verified == other.verified and + self.sync == other.sync and + self.suspicious == other.suspicious + ) + return False + + def __hash__(self): + return hash((self.uid, self.hotkey, self.ip, self.version, self.country)) diff --git a/tests/unit_tests/mocks/mock_country.py b/tests/unit_tests/mocks/mock_country.py new file mode 100644 index 00000000..0716be92 --- /dev/null +++ b/tests/unit_tests/mocks/mock_country.py @@ -0,0 +1,6 @@ +def mock_get_country(mock_get_country, axon_details): + def side_effect(*args): + country = next((x["country"] for x in axon_details if x["ip"] == args[0]), None) + return country or "GB" + + mock_get_country.side_effect = side_effect diff --git a/tests/unit_tests/mocks/mock_redis.py b/tests/unit_tests/mocks/mock_redis.py index e723aa5a..a053d1e6 100644 --- a/tests/unit_tests/mocks/mock_redis.py +++ b/tests/unit_tests/mocks/mock_redis.py @@ -14,3 +14,16 @@ def mock_get_selection(hoktkey: str, selection: List[int] = None): ) return mocked_redis + + +def mock_get_statistics(hoktkeys: List[str]): + # Mock the redis instance + mocked_redis = AsyncMock() + + # Set the return value for redis.get + selection_keys = [f"stats:{hotkey}" for hotkey in hoktkeys] + mocked_redis.hgetall = AsyncMock( + side_effect=lambda key: (None if key in selection_keys else None) + ) + + return mocked_redis diff --git a/tests/unit_tests/subnet/test_get_next_uids.py b/tests/unit_tests/subnet/test_get_next_uids.py index e0bff961..4960fc5f 100644 --- a/tests/unit_tests/subnet/test_get_next_uids.py +++ b/tests/unit_tests/subnet/test_get_next_uids.py @@ -3,8 +3,8 @@ from subnet.constants import DEFAULT_CHUNK_SIZE from subnet.validator.utils import get_next_uids from tests.unit_tests.mocks import mock_redis -from tests.unit_tests.utils import generate_random_ip -from tests.unit_tests.utils import count_non_unique, count_unique +from tests.unit_tests.utils.utils import generate_random_ip +from tests.unit_tests.utils.utils import count_non_unique, count_unique @pytest.mark.asyncio diff --git a/tests/unit_tests/subnet/test_get_uids_selection.py b/tests/unit_tests/subnet/test_get_uids_selection.py index 68e78b31..078cfee5 100644 --- a/tests/unit_tests/subnet/test_get_uids_selection.py +++ b/tests/unit_tests/subnet/test_get_uids_selection.py @@ -1,6 +1,6 @@ from subnet.validator.selection import select_uids from tests.unit_tests.mocks import mock_miners -from tests.unit_tests.utils import count_non_unique, count_unique +from tests.unit_tests.utils.utils import count_non_unique, count_unique # List of 18 miners from uid 0 to 17 miners = sorted( diff --git a/tests/unit_tests/subnet/validator/test_get_available_query_miners.py b/tests/unit_tests/subnet/validator/test_get_available_query_miners.py index 641ee3c9..0f81d4e6 100644 --- a/tests/unit_tests/subnet/validator/test_get_available_query_miners.py +++ b/tests/unit_tests/subnet/validator/test_get_available_query_miners.py @@ -1,6 +1,6 @@ from subnet.validator.utils import get_available_query_miners -from tests.unit_tests.utils import generate_random_ip +from tests.unit_tests.utils.utils import generate_random_ip def test_querying_3_miners_without_exclusion_should_return_a_list_of_3_miners( diff --git a/tests/unit_tests/subnet/validator/test_get_available_uids.py b/tests/unit_tests/subnet/validator/test_get_available_uids.py index 85a08cef..ad17623b 100644 --- a/tests/unit_tests/subnet/validator/test_get_available_uids.py +++ b/tests/unit_tests/subnet/validator/test_get_available_uids.py @@ -1,6 +1,6 @@ from subnet.validator.utils import get_available_uids -from tests.unit_tests.utils import generate_random_ip +from tests.unit_tests.utils.utils import generate_random_ip def test_given_a_list_of_uids_without_exclusion_when_all_uids_are_available_should_return_all_the_uids( diff --git a/tests/unit_tests/subnet/validator/test_resync_miners.py b/tests/unit_tests/subnet/validator/test_resync_miners.py new file mode 100644 index 00000000..d84ffb8d --- /dev/null +++ b/tests/unit_tests/subnet/validator/test_resync_miners.py @@ -0,0 +1,294 @@ +import copy +import pytest +import unittest +from unittest.mock import patch + +from tests.unit_tests.mocks import mock_redis, mock_country +from tests.unit_tests.utils.metagraph import ( + sync_metagraph, + add_new_miner, + move_miner, + replace_old_miner, + remove_miner, +) + +from subnet.validator.miner import resync_miners, get_all_miners + +default_axons_details = [ + {"ip": "23.244.235.121", "country": "US"}, + {"ip": "55.228.3.149", "country": "US"}, + {"ip": "43.82.230.186", "country": "SG"}, + {"ip": "191.230.100.214", "country": "BR"}, + {"ip": "85.62.110.203", "country": "ES"}, + {"ip": "187.64.109.14", "country": "BR"}, + {"ip": "38.75.105.111", "country": "KR"}, + {"ip": "176.65.235.230", "country": "IR"}, + {"ip": "34.20.248.3", "country": "US"}, + {"ip": "35.131.97.24", "country": "US"}, + {"ip": "38.213.246.7", "country": "US"}, + {"ip": "89.116.159.53", "country": "US"}, + {"ip": "9.91.241.47", "country": "US"}, + {"ip": "70.229.181.61", "country": "US"}, + {"ip": "88.30.74.99", "country": "ES"}, + {"ip": "88.30.24.99", "country": "ES"}, + {"ip": "9.91.241.48", "country": "ES"}, +] + + +class TestResyncMiners(unittest.IsolatedAsyncioTestCase): + @pytest.fixture(autouse=True) + def prepare_fixture(self, validator): + self.validator = validator + + def setUp(self): + self.chain_state = copy.deepcopy(self.validator.subtensor.chain_state) + + def tearDown(self): + self.validator.subtensor.chain_state = self.chain_state + sync_metagraph(self.validator, default_axons_details) + + @patch("subnet.validator.miner.get_country") + async def test_given_a_metagraph_when_no_change_should_return_the_same_list_of_miners( + self, + mock_get_country, + ): + # Arrange + axons_details = copy.deepcopy(default_axons_details) + + axons = self.validator.metagraph.axons + for idx, axon in enumerate(axons): + axon.ip = axons_details[idx]["ip"] + + mock_country.mock_get_country(mock_get_country, axons_details) + self.validator.database = mock_redis.mock_get_statistics( + self.validator.metagraph.hotkeys + ) + + sync_metagraph(self.validator, axons_details) + miners = await get_all_miners(self.validator) + self.validator.miners = copy.deepcopy(miners) + + # Act + await resync_miners(self.validator) + + # Assert + assert miners == self.validator.miners + + @patch("subnet.validator.miner.get_country") + async def test_given_a_partially_full_metagraph_when_a_new_neuron_is_added_should_be_added_to_the_list( + self, + mock_get_country, + ): + # Arrange + axons_details = copy.deepcopy(default_axons_details) + [ + {"ip": "19.91.241.48", "country": "US"} + ] + + mock_country.mock_get_country(mock_get_country, axons_details) + self.validator.database = mock_redis.mock_get_statistics( + self.validator.metagraph.hotkeys + ) + + sync_metagraph(self.validator, axons_details) + miners = await get_all_miners(self.validator) + self.validator.miners = copy.deepcopy(miners) + + uid = add_new_miner(self.validator, axons_details) + + # Act + await resync_miners(self.validator) + + # Assert + miner = next( + (element for element in self.validator.miners if element.uid == uid), None + ) + + assert len(miners) + 1 == len(self.validator.miners) + assert miners == self.validator.miners[:-1] + + miner = self.validator.miners[len(self.validator.miners) - 1] + assert 17 == miner.uid + assert "miner-hotkey-17" == miner.hotkey + assert "19.91.241.48" == miner.ip + assert 1 == miner.ip_occurences + assert "0.0.0" == miner.version + assert "US" == miner.country + assert 0 == miner.challenge_successes + assert 0 == miner.challenge_successes + assert 0 == miner.availability_score + assert 0 == miner.distribution_score + assert 0 == miner.reliability_score + assert 0 == miner.latency_score + assert 0 == miner.score + assert 0 == miner.process_time + assert False == miner.suspicious + assert False == miner.sync + assert False == miner.verified + + @patch("subnet.validator.miner.get_country") + async def test_given_a_full_metagraph_when_a_uid_has_a_new_hotkey_with_same_ip_should_replace_the_old_miner_by_the_new_one_in_the_list( + self, + mock_get_country, + ): + # Arrange + axons_details = copy.deepcopy(default_axons_details) + + mock_country.mock_get_country(mock_get_country, axons_details) + self.validator.database = mock_redis.mock_get_statistics( + self.validator.metagraph.hotkeys + ) + + sync_metagraph(self.validator, axons_details) + miners = await get_all_miners(self.validator) + self.validator.miners = copy.deepcopy(miners) + + new_uid = replace_old_miner( + self.validator, + axons_details, + ) + + # Act + await resync_miners(self.validator) + + # Assert + assert len(miners) == len(self.validator.miners) + + axon_detail = axons_details[new_uid] + miner = self.validator.miners[new_uid] + assert "miner-hotkey-17" == miner.hotkey + assert axon_detail["ip"] == miner.ip + assert 1 == miner.ip_occurences + assert "0.0.0" == miner.version + assert axon_detail["country"] == miner.country + assert 0 == miner.challenge_successes + assert 0 == miner.challenge_successes + assert 0 == miner.availability_score + assert 0 == miner.distribution_score + assert 0 == miner.reliability_score + assert 0 == miner.latency_score + assert 0 == miner.score + assert 0 == miner.process_time + assert False == miner.suspicious + assert False == miner.sync + assert False == miner.verified + + @patch("subnet.validator.miner.get_country") + async def test_given_a_full_metagraph_when_a_uid_has_a_same_hotkey_with_different_ip_should_replace_the_old_miner_by_the_new_one_in_the_list( + self, + mock_get_country, + ): + # Arrange + axons_details = copy.deepcopy(default_axons_details) + + mock_country.mock_get_country(mock_get_country, axons_details) + self.validator.database = mock_redis.mock_get_statistics( + self.validator.metagraph.hotkeys + ) + + sync_metagraph(self.validator, axons_details) + miners = await get_all_miners(self.validator) + self.validator.miners = copy.deepcopy(miners) + + uid = move_miner( + self.validator, + axons_details, + 10, + axon_detail={"ip": "31.129.22.101", "country": "PT"}, + ) + + # Act + await resync_miners(self.validator) + + # Assert + assert len(miners) == len(self.validator.miners) + + miner = self.validator.miners[uid] + assert f"miner-hotkey-{uid}" == miner.hotkey + assert "31.129.22.101" == miner.ip + assert 1 == miner.ip_occurences + assert "0.0.0" == miner.version + assert "PT" == miner.country + assert 0 == miner.challenge_successes + assert 0 == miner.challenge_successes + assert 0 == miner.availability_score + assert 0 == miner.distribution_score + assert 0 == miner.reliability_score + assert 0 == miner.latency_score + assert 0 == miner.score + assert 0 == miner.process_time + assert False == miner.suspicious + assert False == miner.sync + assert False == miner.verified + + @patch("subnet.validator.miner.get_country") + async def test_given_a_full_metagraph_when_a_uid_has_a_new_hotkey_with_different_ip_should_replace_the_old_miner_by_the_new_one_in_the_list( + self, + mock_get_country, + ): + # Arrange + axons_details = copy.deepcopy(default_axons_details) + + mock_country.mock_get_country(mock_get_country, axons_details) + self.validator.database = mock_redis.mock_get_statistics( + self.validator.metagraph.hotkeys + ) + + sync_metagraph(self.validator, axons_details) + miners = await get_all_miners(self.validator) + self.validator.miners = copy.deepcopy(miners) + + new_uid = replace_old_miner( + self.validator, + axons_details, + axon_detail={"ip": "19.91.241.48", "country": "US"}, + ) + + # Act + await resync_miners(self.validator) + + # Assert + assert len(miners) == len(self.validator.miners) + + miner = self.validator.miners[new_uid] + assert "miner-hotkey-17" == miner.hotkey + assert "19.91.241.48" == miner.ip + assert 1 == miner.ip_occurences + assert "0.0.0" == miner.version + assert "US" == miner.country + assert 0 == miner.challenge_successes + assert 0 == miner.challenge_successes + assert 0 == miner.availability_score + assert 0 == miner.distribution_score + assert 0 == miner.reliability_score + assert 0 == miner.latency_score + assert 0 == miner.score + assert 0 == miner.process_time + assert False == miner.suspicious + assert False == miner.sync + assert False == miner.verified + + @patch("subnet.validator.miner.get_country") + async def test_given_a_metagraph_when_a_uid_is_not_running_should_be_removed_from_the_list( + self, + mock_get_country, + ): + # Arrange + axons_details = copy.deepcopy(default_axons_details) + + mock_country.mock_get_country(mock_get_country, axons_details) + self.validator.database = mock_redis.mock_get_statistics( + self.validator.metagraph.hotkeys + ) + + sync_metagraph(self.validator, axons_details) + miners = await get_all_miners(self.validator) + self.validator.miners = copy.deepcopy(miners) + + uid = remove_miner(self.validator, axons_details, 15) + + # Act + await resync_miners(self.validator) + + # Assert + assert len(miners) - 1 == len(self.validator.miners) + assert False == any(element.uid == uid for element in self.validator.miners) diff --git a/tests/unit_tests/utils/__init__.py b/tests/unit_tests/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit_tests/utils/metagraph.py b/tests/unit_tests/utils/metagraph.py new file mode 100644 index 00000000..4dfd9fc4 --- /dev/null +++ b/tests/unit_tests/utils/metagraph.py @@ -0,0 +1,77 @@ +from neurons.validator import Validator + + +def sync_metagraph(validator: Validator, axon_details): + # Sync the metagraph + validator.metagraph.sync(subtensor=validator.subtensor) + + # Refresh ip + axons = validator.metagraph.axons + for idx, axon in enumerate(axons): + axon.ip = axon_details[idx]["ip"] + + +def add_new_miner(validator: Validator, axon_details): + """ + Add a new miner to the metagraph + """ + n = validator.subtensor.chain_state["SubtensorModule"]["SubnetworkN"][7][0] + + # Add new neurons + uid = validator.subtensor.force_register_neuron( + netuid=7, + hotkey=f"miner-hotkey-{n}", + coldkey="mock-coldkey", + balance=100000, + stake=100000, + ) + + sync_metagraph(validator, axon_details) + + return uid + + +def replace_old_miner(validator: Validator, axon_details, axon_detail=None): + """ + Replace a old miner by new one + """ + # Get the number of neurons + n = validator.subtensor.chain_state["SubtensorModule"]["SubnetworkN"][7][0] + + # Set the max to the current number + validator.subtensor.chain_state["SubtensorModule"]["MaxAllowedUids"][7] = {0: n} + + # Add new neurons + uid = validator.subtensor.force_register_neuron( + netuid=7, + hotkey=f"miner-hotkey-{n}", + coldkey="mock-coldkey", + balance=100000, + stake=100000, + ) + + axon_details[uid] = axon_detail or axon_details[uid] + sync_metagraph(validator, axon_details) + + return uid + + +def move_miner(validator: Validator, axon_details, uid, axon_detail=None): + """ + Move a miner from an ip to another one + """ + axon_details[uid] = axon_detail or axon_details[uid] + sync_metagraph(validator, axon_details) + + return uid + + +def remove_miner(validator: Validator, axon_details, uid): + """ + Remove a miner from the metagraph + """ + axon_details[uid]['ip'] = "0.0.0.0" + + sync_metagraph(validator, axon_details) + + return uid diff --git a/tests/unit_tests/utils.py b/tests/unit_tests/utils/utils.py similarity index 100% rename from tests/unit_tests/utils.py rename to tests/unit_tests/utils/utils.py From dadca31d418c520654b4ef56b47f462dde4a1b1c Mon Sep 17 00:00:00 2001 From: eclipsevortex Date: Mon, 6 May 2024 09:28:44 +0100 Subject: [PATCH 2/5] implement auto upgrade --- README.md | 6 +- neurons/miner.py | 3 + neurons/validator.py | 28 +- scripts/redis/README.md | 68 +++ scripts/redis/migrations/migration-2.2.0.py | 43 ++ scripts/redis/utils/redis_dump.py | 98 ++++ scripts/redis/utils/redis_migration.py | 123 ++++ scripts/subnet/README.md | 36 ++ scripts/subnet/utils/subnet_upgrade.py | 122 ++++ subnet/miner/config.py | 12 +- subnet/miner/run.py | 18 + subnet/miner/version.py | 64 +++ subnet/shared/utils.py | 10 + subnet/shared/version.py | 67 +++ subnet/validator/config.py | 8 + subnet/validator/database.py | 112 ++++ subnet/validator/state.py | 20 +- subnet/validator/utils.py | 5 +- subnet/validator/version.py | 169 ++++++ subnet/version/github_controller.py | 65 +++ subnet/version/interpreter_controller.py | 18 + subnet/version/redis_controller.py | 107 ++++ subnet/version/utils.py | 102 ++++ subnet/version/version_control.py | 93 +++ tests/unit_tests/mocks/mock_interpreter.py | 8 + tests/unit_tests/mocks/mock_redis.py | 10 + .../miner/test_miner_version_control.py | 108 ++++ .../test_validator_version_control.py | 528 ++++++++++++++++++ .../subnet/version/test_get_migrations.py | 56 ++ 29 files changed, 2088 insertions(+), 19 deletions(-) create mode 100644 scripts/redis/migrations/migration-2.2.0.py create mode 100644 scripts/redis/utils/redis_dump.py create mode 100644 scripts/redis/utils/redis_migration.py create mode 100644 scripts/subnet/utils/subnet_upgrade.py create mode 100644 subnet/miner/version.py create mode 100644 subnet/shared/version.py create mode 100644 subnet/validator/version.py create mode 100644 subnet/version/github_controller.py create mode 100644 subnet/version/interpreter_controller.py create mode 100644 subnet/version/redis_controller.py create mode 100644 subnet/version/utils.py create mode 100644 subnet/version/version_control.py create mode 100644 tests/unit_tests/mocks/mock_interpreter.py create mode 100644 tests/unit_tests/subnet/miner/test_miner_version_control.py create mode 100644 tests/unit_tests/subnet/validator/test_validator_version_control.py create mode 100644 tests/unit_tests/subnet/version/test_get_migrations.py diff --git a/README.md b/README.md index a28a4972..973250bb 100644 --- a/README.md +++ b/README.md @@ -345,7 +345,8 @@ pm2 start neurons/miner.py \ --subtensor.network local \ --wallet.name YOUR_WALLET_NAME \ --wallet.hotkey YOUR_HOTKEY_NAME \ - --logging.debug + --logging.debug \ + --auto-update ``` > IMPORTANT: Do not run more than one miner per machine. Running multiple miners will result in the loss of incentive and emissions on all miners. @@ -367,7 +368,8 @@ pm2 start neurons/validator.py \ --netuid \ --wallet.name YOUR_WALLET_NAME \ --wallet.hotkey YOUR_HOTKEY_NAME \ - --logging.debug + --logging.debug \ + --auto-update ``` > NOTE: if you run a validator in testnet do not forget to add the argument `--subtensor.network test` or `--subtensor.chain_endpoint ws://:9944` (the local subtensor has to target the network testnet) diff --git a/neurons/miner.py b/neurons/miner.py index a09f17aa..374fdf39 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -86,6 +86,9 @@ def __init__(self): bt.logging(config=self.config, logging_dir=self.config.miner.full_path) bt.logging.info(f"{self.config}") + # Show miner version + bt.logging.debug(f"miner version {THIS_VERSION}") + # Init device. bt.logging.debug("loading device") self.device = torch.device(self.config.miner.device) diff --git a/neurons/validator.py b/neurons/validator.py index 957f996d..9c5b5860 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -25,10 +25,12 @@ from typing import List from traceback import print_exception +from subnet import __version__ as THIS_VERSION + from subnet.monitor.monitor import Monitor from subnet.shared.checks import check_registration -from subnet.shared.utils import get_redis_password +from subnet.shared.utils import get_redis_password, should_upgrade from subnet.shared.subtensor import get_current_block from subnet.shared.weights import should_set_weights from subnet.shared.mock import MockMetagraph, MockDendrite, MockSubtensor @@ -37,13 +39,14 @@ from subnet.validator.localisation import get_country, get_localisation from subnet.validator.forward import forward from subnet.validator.models import Miner +from subnet.validator.version import VersionControl from subnet.validator.miner import get_all_miners from subnet.validator.state import ( resync_metagraph_and_miners, load_state, save_state, init_wandb, - reinit_wandb, + finish_wandb, should_reinit_wandb, ) from subnet.validator.weights import ( @@ -88,6 +91,9 @@ def __init__(self, config=None): self.check_config(self.config) bt.logging(config=self.config, logging_dir=self.config.neuron.full_path) + # Show miner version + bt.logging.debug(f"validator version {THIS_VERSION}") + # Init device. bt.logging.debug("loading device") self.device = torch.device(self.config.neuron.device) @@ -181,10 +187,14 @@ def __init__(self, config=None): self.last_registered_block = 0 self.rebalance_queue = [] self.miners: List[Miner] = [] + self.last_upgrade_check = 0 async def run(self): bt.logging.info("run()") + # Initi versioin control + self.version_control = VersionControl(self.database) + # Init miners self.miners = await get_all_miners(self) bt.logging.debug(f"Miners loaded {len(self.miners)}") @@ -256,10 +266,22 @@ async def run_forward(): prev_set_weights_block = get_current_block(self.subtensor) save_state(self) + # Start the upgrade process every 10 minutes + if should_upgrade(self.config.auto_update, self.last_upgrade_check): + bt.logging.debug("Checking upgrade") + must_restart = await self.version_control.upgrade() + if must_restart: + finish_wandb() + self.version_control.restart() + return + + self.last_upgrade_check = time.time() + # Rollover wandb to a new run. if should_reinit_wandb(self): bt.logging.info("Reinitializing wandb") - reinit_wandb(self) + finish_wandb() + init_wandb(self) self.prev_step_block = get_current_block(self.subtensor) if self.config.neuron.verbose: diff --git a/scripts/redis/README.md b/scripts/redis/README.md index 96daae91..e401c9cf 100644 --- a/scripts/redis/README.md +++ b/scripts/redis/README.md @@ -12,6 +12,12 @@ This document explains how to install and uninstall a redis. - [Uninstallation](#uninstallation) - [As process](#uninstallation-as-process) - [As docker container](#uninstallation-as-container) +- [Migration](#migration) + - [Rollout](#migration-rollout) + - [Rollback](#migration-rollback) +- [Dump](#migration) + - [Creation](#dump-creation) + - [Restoration](#dump-restoration) --- @@ -252,3 +258,65 @@ You shoud have something similar (or at least list that does not container `subv ``` CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES ``` + +# Migration + +## Rollout + +To rollout any Redis migration manually, you can use the python script `redis_migration.py`. + +For example, if you want to rollout the version 2.2.1, you can run in `SubVortex` + +``` +python3 ./scripts/redis/utils/redis_migration.py --run-type rollout --version 2.2.1 +``` + +> IMPORTANT
+> If you have to rollout multiple versions, execute them one by one from your current version to the targeted one. + +## Rollback + +To rollback any Redis migration manually, you can use the python script `redis_migration.py`. + +For example, if you want to rollback the version 2.2.1, you can run in `SubVortex` + +``` +python3 ./scripts/redis/utils/redis_migration.py --run-type rollback --version 2.2.1 +``` + +> IMPORTANT
+> If you have to rollback multiple versions, execute them one by one from your current version to the targeted one. + +# Dump + +## Creation + +To create a Redis dump manually, you can use the python script `redis_dump.py`. + +For example, if you want to create the dump in the `subVortex` directory, you can run + +``` +python3 ./scripts/redis/utils/redis_dump.py --run-type create +``` + +If you want to create the dump in another location and/or name, you can use the argument `--dump-path` + +``` +python3 ./scripts/redis/utils/redis_dump.py --run-type create --dump-path /tmp/redis/redis-dump-2.0.0.json +``` + +## Restoration + +To restore a Redis dump manually, you can use the python script `redis_dump.py`. + +For example, if you want to create in `subVortex` directory, you can run + +``` +python3 ./scripts/redis/utils/redis_dump.py --run-type restore +``` + +If you want to restore a dump in another location, you can use the argument `--dump-path` + +``` +python3 ./scripts/redis/utils/redis_dump.py --run-type restore --dump-path /tmp/redis +``` diff --git a/scripts/redis/migrations/migration-2.2.0.py b/scripts/redis/migrations/migration-2.2.0.py new file mode 100644 index 00000000..3533bbca --- /dev/null +++ b/scripts/redis/migrations/migration-2.2.0.py @@ -0,0 +1,43 @@ +from redis import asyncio as aioredis + +current = "2.0.0" + + +async def rollout(database: aioredis.Redis): + async for key in database.scan_iter("stats:*"): + metadata_dict = await database.hgetall(key) + + if b"subtensor_successes" not in metadata_dict: + await database.hset(key, b"subtensor_successes", 0) + if b"subtensor_attempts" not in metadata_dict: + await database.hset(key, b"subtensor_attempts", 0) + if b"metric_successes" not in metadata_dict: + await database.hset(key, b"metric_successes", 0) + if b"metric_attempts" not in metadata_dict: + await database.hset(key, b"metric_attempts", 0) + if b"total_successes" not in metadata_dict: + await database.hset(key, b"total_successes", 0) + if b"tier" not in metadata_dict: + await database.hset(key, b"tier", "Bronze") + + await database.set("version", current) + + +async def rollback(database: aioredis.Redis): + async for key in database.scan_iter("stats:*"): + metadata_dict = await database.hgetall(key) + + if b"subtensor_successes" in metadata_dict: + await database.hdel(key, b"subtensor_successes") + if b"subtensor_attempts" in metadata_dict: + await database.hdel(key, b"subtensor_attempts") + if b"metric_successes" in metadata_dict: + await database.hdel(key, b"metric_successes") + if b"metric_attempts" in metadata_dict: + await database.hdel(key, b"metric_attempts") + if b"total_successes" in metadata_dict: + await database.hdel(key, b"total_successes") + if b"tier" in metadata_dict: + await database.hdel(key, b"tier") + + await database.set("version", None) diff --git a/scripts/redis/utils/redis_dump.py b/scripts/redis/utils/redis_dump.py new file mode 100644 index 00000000..dc7da88f --- /dev/null +++ b/scripts/redis/utils/redis_dump.py @@ -0,0 +1,98 @@ +import asyncio +import argparse +import bittensor as bt +from redis import asyncio as aioredis + +from subnet.shared.utils import get_redis_password +from subnet.validator.database import create_dump, restore_dump + + +async def create(args): + try: + bt.logging.info( + f"Loading database from {args.database_host}:{args.database_port}" + ) + redis_password = get_redis_password(args.redis_password) + database = aioredis.StrictRedis( + host=args.database_host, + port=args.database_port, + db=args.database_index, + password=redis_password, + ) + + bt.logging.info("Create dump starting") + + await create_dump(args.dump_path, database) + + bt.logging.success("Create dump successful") + except Exception as e: + bt.logging.error(f"Error during rollout: {e}") + + +async def restore(args): + try: + bt.logging.info( + f"Loading database from {args.database_host}:{args.database_port}" + ) + redis_password = get_redis_password(args.redis_password) + database = aioredis.StrictRedis( + host=args.database_host, + port=args.database_port, + db=args.database_index, + password=redis_password, + ) + + bt.logging.info("Restore dump starting") + + await restore_dump(args.dump_path, database) + + bt.logging.success("Restore dump successful") + + except Exception as e: + bt.logging.error(f"Error during rollback: {e}") + + +async def main(args): + if args.run_type == "create": + await create(args) + else: + await restore(args) + + +if __name__ == "__main__": + try: + parser = argparse.ArgumentParser() + parser.add_argument( + "--run-type", + type=str, + default="create", + help="Type of migration you want too execute. Possible values are rollout or rollback)", + ) + parser.add_argument( + "--dump-path", + type=str, + default="", + help="Dump file (with path) to create or restore", + ) + parser.add_argument( + "--redis_password", + type=str, + default=None, + help="password for the redis database", + ) + parser.add_argument( + "--redis_conf_path", + type=str, + default="/etc/redis/redis.conf", + help="path to the redis configuration file", + ) + parser.add_argument("--database_host", type=str, default="localhost") + parser.add_argument("--database_port", type=int, default=6379) + parser.add_argument("--database_index", type=int, default=1) + args = parser.parse_args() + + asyncio.run(main(args)) + except KeyboardInterrupt: + print("KeyboardInterrupt") + except ValueError as e: + print(f"ValueError: {e}") diff --git a/scripts/redis/utils/redis_migration.py b/scripts/redis/utils/redis_migration.py new file mode 100644 index 00000000..3edb2154 --- /dev/null +++ b/scripts/redis/utils/redis_migration.py @@ -0,0 +1,123 @@ +import asyncio +import argparse +import importlib +import bittensor as bt +from redis import asyncio as aioredis + +from subnet.shared.utils import get_redis_password + + +def get_migration(version): + file_path = f"scripts/redis/migrations/migration-{version}.py" + spec = importlib.util.spec_from_file_location("migration_module", file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +async def rollout(args): + try: + bt.logging.info( + f"Loading database from {args.database_host}:{args.database_port}" + ) + redis_password = get_redis_password(args.redis_password) + database = aioredis.StrictRedis( + host=args.database_host, + port=args.database_port, + db=args.database_index, + password=redis_password, + ) + + bt.logging.info("Rollout starting") + + # Get the migration + migration = get_migration(args.version) + if not migration: + bt.logging.error(f"Could not find the migration {args.version}") + return + + # Rollback the migration + await migration.rollout(database) + + bt.logging.success("Rollout successful") + except Exception as e: + bt.logging.error(f"Error during rollout: {e}") + + +async def rollback(args): + try: + bt.logging.info( + f"Loading database from {args.database_host}:{args.database_port}" + ) + redis_password = get_redis_password(args.redis_password) + database = aioredis.StrictRedis( + host=args.database_host, + port=args.database_port, + db=args.database_index, + password=redis_password, + ) + + bt.logging.info("Rollback starting") + + # Get the migration + migration = get_migration(args.version) + if not migration: + bt.logging.error(f"Could not find the migration {args.version}") + return + + # Rollback the migration + await migration.rollback(database) + + bt.logging.success("Rollback successful") + + except Exception as e: + bt.logging.error(f"Error during rollback: {e}") + + +async def main(args): + if not args.version: + bt.logging.error(f"Version is not provided") + return + + if args.run_type == "rollout": + await rollout(args) + else: + await rollback(args) + + +if __name__ == "__main__": + try: + parser = argparse.ArgumentParser() + parser.add_argument( + "--run-type", + type=str, + default="rollout", + help="Type of migration you want too execute. Possible values are rollout or rollback)", + ) + parser.add_argument( + "--version", + type=str, + help="Verstion to run", + ) + parser.add_argument( + "--redis_password", + type=str, + default=None, + help="password for the redis database", + ) + parser.add_argument( + "--redis_conf_path", + type=str, + default="/etc/redis/redis.conf", + help="path to the redis configuration file", + ) + parser.add_argument("--database_host", type=str, default="localhost") + parser.add_argument("--database_port", type=int, default=6379) + parser.add_argument("--database_index", type=int, default=1) + args = parser.parse_args() + + asyncio.run(main(args)) + except KeyboardInterrupt: + print("KeyboardInterrupt") + except ValueError as e: + print(f"ValueError: {e}") diff --git a/scripts/subnet/README.md b/scripts/subnet/README.md index 610e91a7..53697522 100644 --- a/scripts/subnet/README.md +++ b/scripts/subnet/README.md @@ -2,6 +2,20 @@ This document explains how to install and uninstall the subnet SubVortex. +
+ +--- + +- [Installation](#intasllation) +- [Uninstallation](#uninstallation) +- [Migration](#migration) + - [Migrate](#migration-migrate) + - [Downgrade](#migration-downgrade) + +--- + +
+ # Installation Before installing the subnet, you have to install some prerequisites @@ -48,3 +62,25 @@ To uninstall the subnet, you can run ``` Be sure you are in the **SubVortex's** parent directory + +# Migration + +## Migrate + +To uprade the Subnet manually, you can use the python script `subnet_upgrade.py`. + +For example, if you are on tag v2.2.2 and want to migrate to the tag v2.2.3, you can run in `SubVortex` + +``` +python3 ./scripts/subnet/subnet_upgrade.py --tag v2.2.3 +``` + +## Downgrade + +To downgrade the Subnet manually, you can use the python script `subnet_upgrade.py`. + +For example, if you are on tag v2.2.3 and want to downgrade to the tag v2.2.2, you can run in `SubVortex` + +``` +python3 ./scripts/subnet/subnet_upgrade.py --tag v2.2.2 +``` diff --git a/scripts/subnet/utils/subnet_upgrade.py b/scripts/subnet/utils/subnet_upgrade.py new file mode 100644 index 00000000..2a36acfc --- /dev/null +++ b/scripts/subnet/utils/subnet_upgrade.py @@ -0,0 +1,122 @@ +import asyncio +import argparse +import subprocess +import bittensor as bt + + +def get_current_branch(): + result = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def get_current_tag(): + result = subprocess.run( + ["git", "--tags", "--exact-match"], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def get_current_branch_or_tag(): + try: + tag = get_current_tag() + return ("tag", tag) + except subprocess.CalledProcessError: + branch = get_current_branch() + return ("branch", branch) + + +def get_tags(): + subprocess.run(["git", "fetch", "--tags", "--force"], check=True) + bt.logging.info(f"Fetch tags.") + + +def upgrade_to_tag(tag: str): + # Get current branch or tag + _, tag_or_branch = get_current_branch_or_tag() + + # Check if the requested tag is already pulled + if tag_or_branch == tag: + bt.logging.warning(f"The tag {tag} is already pulled") + return + + # Stash if there is any local changes just in case + subprocess.run(["git", "stash"], check=True) + + # Fetch all tags + subprocess.run(["git", "fetch", "--tags", "--force"], check=True) + bt.logging.debug(f"Fetch tags") + + # Pull the requested tag + subprocess.run(["git", "checkout", f"tags/{tag}"], check=True) + bt.logging.success(f"Successfully pulled source code for tag '{tag}'.") + + +def upgrade_to_branch(branch: str): + # Get current branch or tag + _, tag_or_branch = get_current_branch_or_tag() + + # Check if the requested tag is already pulled + if tag_or_branch == branch: + bt.logging.warning(f"The branch {branch} is already pulled") + return + + # Stash if there is any local changes just in case + subprocess.run(["git", "stash"], check=True) + + # Checkout the branch + subprocess.run(["git", "checkout", "-B", branch], check=True) + bt.logging.debug(f"Checkout the branch {branch}") + + # Pull the branch + subprocess.run(["git", "pull"], check=True) + bt.logging.debug(f"Pull the branch {branch}") + + bt.logging.success(f"Successfully pulled source code for {branch} branch'.") + + +def main(args): + if not args.tag and not args.branch: + bt.logging.error(f"Please provide a tag or a branch to upgrade to") + return + + if args.tag: + upgrade_to_tag(args.tag) + else: + upgrade_to_branch(args.branch) + + subprocess.run(["pip", "install", "-r", "requirements.txt"]) + bt.logging.info(f"Dependencies installed successfully") + + subprocess.run(["pip", "install", "-e", "."]) + bt.logging.info(f"Source installed successfully") + + +if __name__ == "__main__": + try: + parser = argparse.ArgumentParser() + parser.add_argument( + "--tag", + type=str, + help="Tag to pull", + ) + parser.add_argument( + "--branch", + type=str, + default="main", + help="Branch to pull", + ) + args = parser.parse_args() + + main(args) + except KeyboardInterrupt: + print("KeyboardInterrupt") + except ValueError as e: + print(f"ValueError: {e}") diff --git a/subnet/miner/config.py b/subnet/miner/config.py index cd214652..decc8dab 100644 --- a/subnet/miner/config.py +++ b/subnet/miner/config.py @@ -104,12 +104,12 @@ def check_config(cls, config: "bt.Config"): def add_args(cls, parser): - parser.add_argument("--netuid", type=int, default=21, help="The chain subnet uid.") + parser.add_argument("--netuid", type=int, help="Subvortex network netuid", default=7) parser.add_argument( "--miner.name", type=str, help="Trials for this miner go in miner.root / (wallet_cold - wallet_hot) / miner.name. ", - default="core_storage_miner", + default="subvortex_miner", ) parser.add_argument( "--miner.device", @@ -125,6 +125,14 @@ def add_args(cls, parser): default="requests_log.json", ) + # Auto update + parser.add_argument( + "--auto-update", + action="store_true", + help="True if the miner can be auto updated, false otherwise", + default=True, + ) + # Mocks. parser.add_argument( "--miner.mock_subtensor", diff --git a/subnet/miner/run.py b/subnet/miner/run.py index 0585a44d..ad444ac4 100644 --- a/subnet/miner/run.py +++ b/subnet/miner/run.py @@ -1,7 +1,11 @@ +import time import bittensor as bt from substrateinterface import SubstrateInterface + from subnet.shared.checks import check_registration +from subnet.shared.utils import should_upgrade +from subnet.miner.version import VersionControl def run(self): """ @@ -37,6 +41,11 @@ def run(self): netuid = self.config.netuid + version_control = VersionControl() + + # Keep a track of last upgrade check + self.last_upgrade_check = 0 + # --- Check for registration. check_registration(self.subtensor, self.wallet, netuid) @@ -52,6 +61,15 @@ def handler(obj, update_nr, subscription_id): if current_block % 100 == 0: check_registration(self.subtensor, self.wallet, netuid) + if should_upgrade(self.config.auto_update, self.last_upgrade_check): + bt.logging.debug("Checking upgrade") + must_restart = version_control.upgrade() + if must_restart: + self.version_control.restart() + return + + self.last_upgrade_check = time.time() + bt.logging.debug( f"Blocks since epoch: {(current_block + netuid + 1) % (tempo + 1)}" ) diff --git a/subnet/miner/version.py b/subnet/miner/version.py new file mode 100644 index 00000000..9adc1002 --- /dev/null +++ b/subnet/miner/version.py @@ -0,0 +1,64 @@ +import os +import sys +import bittensor as bt + +from subnet.shared.version import BaseVersionControl + + +class VersionControl(BaseVersionControl): + def restart(self): + bt.logging.info(f"Restarting miner...") + python = sys.executable + os.execl(python, python, *sys.argv) + + def upgrade_miner(self): + current_version = None + + try: + # Get the remote version + remote_version = self.github.get_latest_version() + bt.logging.debug(f"[Subnet] Remote version: {remote_version}") + + # Get the local version + current_version = self.github.get_version() + bt.logging.debug(f"[Subnet] Local version: {current_version}") + + # Check if the subnet has to be upgraded + if current_version == remote_version: + bt.logging.success(f"[Subnet] Already using {current_version}") + return + + self.must_restart = True + + current_version_num = int(current_version.replace(".", "")) + remote_version_num = int(remote_version.replace(".", "")) + + if remote_version_num > current_version_num: + success = self.upgrade_subnet(remote_version) + else: + success = self.downgrade_subnet(remote_version) + + if not success: + self.downgrade_subnet(current_version) + except Exception as err: + bt.logging.error(f"[Subnet] Upgrade failed: {err}") + bt.logging.info(f"[Subnet] Rolling back to {current_version}...") + self.downgrade_subnet(current_version) + + def upgrade(self): + try: + # Flag to stop miner activity + self.upgrading = True + + # Upgrade subnet + self.upgrade_miner() + + return self.must_restart + except Exception as ex: + bt.logging.error(f"Could not upgrade the miner: {ex}") + finally: + # Unflag to resume miner activity + self.upgrading = False + self.must_restart = False + + return True diff --git a/subnet/shared/utils.py b/subnet/shared/utils.py index 250ad97b..0da04823 100644 --- a/subnet/shared/utils.py +++ b/subnet/shared/utils.py @@ -16,6 +16,7 @@ # DEALINGS IN THE SOFTWARE. import os +import time import subprocess import bittensor as bt @@ -43,3 +44,12 @@ def get_redis_password( exit(1) return redis_password + + +def should_upgrade(auto_update: bool, last_upgrade_check: float): + """ + True if it is sime to upgrade, false otherwise + For now, upgrading evering 60 seconds + """ + time_since_last_update = time.time() - last_upgrade_check + return time_since_last_update >= 60 and auto_update diff --git a/subnet/shared/version.py b/subnet/shared/version.py new file mode 100644 index 00000000..d4dd9aa9 --- /dev/null +++ b/subnet/shared/version.py @@ -0,0 +1,67 @@ +import threading +import bittensor as bt + +from subnet.version.github_controller import Github +from subnet.version.interpreter_controller import Interpreter + + +class BaseVersionControl: + _lock = threading.Lock() + + def __init__(self) -> None: + self.github = Github() + self.interpreter = Interpreter() + self._upgrading = False + self.must_restart = False + + @property + def upgrading(self): + with self._lock: + return self._upgrading + + @upgrading.setter + def upgrading(self, value): + with self._lock: + self._upgrading = value + + def upgrade_subnet(self, version: str): + """ + Upgrade the subnet with the requested version or the latest one + Version has to follow the format major.minor.patch + """ + try: + bt.logging.info("[Subnet] Upgrading...") + + # Pull the branch + self.github.get_tag(f"v{version}") + + # Install dependencies + self.interpreter.upgrade_dependencies() + + bt.logging.success(f"[Subnet] Upgrade to {version} successful") + + return True + except Exception as err: + bt.logging.error(f"[Subnet] Failed to upgrade the subnet: {err}") + + return False + + def downgrade_subnet(self, version: str): + """ + Downgrade the subnet with the requested version + Version has to follow the format major.minor.patch + """ + try: + # Pull the branch + self.github.get_tag(f"v{version}") + + # Install dependencies + self.interpreter.upgrade_dependencies() + + bt.logging.success(f"[Subnet] Downgrade to {version} successful") + + return True + except Exception as err: + bt.logging.error(f"[Subnet] Failed to upgrade the subnet: {err}") + + return False diff --git a/subnet/validator/config.py b/subnet/validator/config.py index 7f221de2..7a884a58 100644 --- a/subnet/validator/config.py +++ b/subnet/validator/config.py @@ -220,6 +220,14 @@ def add_args(cls, parser): default="/etc/redis/redis.conf", ) + # Auto update + parser.add_argument( + "--auto-update", + action="store_true", + help="True if the miner can be auto updated, false otherwise", + default=True, + ) + # Wandb args parser.add_argument( "--wandb.off", action="store_true", help="Turn off wandb.", default=False diff --git a/subnet/validator/database.py b/subnet/validator/database.py index e215c563..17e9e7db 100644 --- a/subnet/validator/database.py +++ b/subnet/validator/database.py @@ -14,6 +14,8 @@ # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +import os +import json import bittensor as bt from redis import asyncio as aioredis @@ -95,3 +97,113 @@ async def get_selected_miners(ss58_address: str, database: aioredis.Redis): ) return [] + + +async def get_version(database: aioredis.Redis): + """ + Get the version for the redis database + """ + version = await database.get("version") + return version.decode("utf-8") if version else None + + +async def set_selection(ss58_address: str, selection: str, database: aioredis.Redis): + """ + Set the uids selection to avoid re-selecting them until all of them have been selected at least one + """ + selection_key = f"selection:{ss58_address}" + await database.set(selection_key, selection) + + +async def set_version(version: str, database: aioredis.Redis): + """ + Set the current redis version + """ + await database.set("version", version) + + +async def create_dump(path: str, database: aioredis.Redis): + """ + Create a dump from the database + """ + # Get all keys in the database + keys = await database.keys(f"*") + + dump = {} + + # Use a pipeline to batch key type queries + async with database.pipeline() as pipe: + for key in keys: + # Query key type in the pipeline + pipe.type(key) + + # Execute the pipeline + key_types = await pipe.execute() + + # Process key-value pairs based on key types + for key, key_type in zip(keys, key_types): + key_str = key.decode("utf-8") + if key_type == b"string": + value = await database.get(key) + dump[key_str] = value.decode("utf-8") if value is not None else None + elif key_type == b"hash": + hash_data = await database.hgetall(key) + dump[key_str] = { + field.decode("utf-8"): value.decode("utf-8") + for field, value in hash_data.items() + } + elif key_type == b"list": + list_data = await database.lrange(key, 0, -1) + dump[key_str] = [item.decode("utf-8") for item in list_data] + elif key_type == b"set": + set_data = await database.smembers(key) + dump[key_str] = {member.decode("utf-8") for member in set_data} + elif key_type == b"zset": + zset_data = await database.zrange(key, 0, -1, withscores=True) + dump[key_str] = [ + (member.decode("utf-8"), score) for member, score in zset_data + ] + + # Get the directory path + directory, _ = os.path.split(path) + + # Ensure the directory exists, create it if it doesn't + os.makedirs(directory, exist_ok=True) + + # Save dump file + with open(path, "w") as file: + json.dump(dump, file) + + +async def restore_dump(path: str, database: aioredis.StrictRedis): + """ + Restore the dump into the database + """ + # Flush the database + await database.flushdb() + + # Load the dump + with open(path, "r") as file: + json_data = file.read() + + dump = json.loads(json_data) + + for key, value in dump.items(): + # Determine the data type of the key-value pair + if isinstance(value, bytes): + # For string keys, set the value + await database.set(key, value) + elif isinstance(value, dict): + # For hash keys, sesut all fields and values + await database.hmset(key, value) + elif isinstance(value, list): + # For list keys, push all elements + await database.lpush(key, *value) + elif isinstance(value, set): + # For database keys, add all members + await database.sadd(key, *value) + elif isinstance(value, list) and all( + isinstance(item, tuple) and len(item) == 2 for item in value + ): + # For sorted set keys, add all members with scores + await database.zadd(key, dict(value)) diff --git a/subnet/validator/state.py b/subnet/validator/state.py index cdd5a349..b4b908ce 100644 --- a/subnet/validator/state.py +++ b/subnet/validator/state.py @@ -322,7 +322,7 @@ def log_event(self, uids: List[int], step_length=None): bt.logging.warning(f"log_event() send data to wandb failed: {err}") -def init_wandb(self, reinit=False): +def init_wandb(self): """Starts a new wandb run.""" tags = [ self.wallet.hotkey.ss58_address, @@ -378,7 +378,7 @@ def init_wandb(self, reinit=False): # Create a new run self.wandb = wandb.init( anonymous="allow", - reinit=reinit, + reinit=True, project=project_name, entity=self.config.wandb.entity, config=wandb_config, @@ -449,13 +449,6 @@ def init_wandb(self, reinit=False): ) -def reinit_wandb(self): - """Reinitializes wandb, rolling over the run.""" - if self.wandb is not None: - self.wandb.finish() - init_wandb(self, reinit=True) - - def should_reinit_wandb(self): """Check if wandb run needs to be rolled over.""" return ( @@ -463,3 +456,12 @@ def should_reinit_wandb(self): and self.step and self.step % self.config.wandb.run_step_length == 0 ) + + +def finish_wandb(): + """ + Finish the current wandb run + """ + bt.logging.debug("Finishing wandb run") + wandb.finish() + assert wandb.run is None diff --git a/subnet/validator/utils.py b/subnet/validator/utils.py index 52df2963..3438122e 100644 --- a/subnet/validator/utils.py +++ b/subnet/validator/utils.py @@ -22,7 +22,7 @@ from subnet.constants import DEFAULT_CHUNK_SIZE from subnet.validator.selection import select_uids -from subnet.validator.database import get_selected_miners +from subnet.validator.database import get_selected_miners, set_selection def current_block_hash(self): @@ -260,9 +260,8 @@ async def get_next_uids(self, ss58_address: str, k: int = DEFAULT_CHUNK_SIZE): bt.logging.debug(f"get_next_uids() uids selected: {uids_selected}") # Store the new selection in the database - selection_key = f"selection:{ss58_address}" selection = ",".join(str(uid) for uid in uids_already_selected + uids_selected) - await self.database.set(selection_key, selection) + await set_selection(ss58_address, selection, self.database) bt.logging.debug(f"get_next_uids() new uids selection stored: {selection}") return uids_selected diff --git a/subnet/validator/version.py b/subnet/validator/version.py new file mode 100644 index 00000000..0c9a974b --- /dev/null +++ b/subnet/validator/version.py @@ -0,0 +1,169 @@ +import os +import sys +import bittensor as bt + +from subnet.shared.version import BaseVersionControl +from subnet.version.redis_controller import Redis +from subnet.version.utils import create_dump_migrations, remove_dump_migrations +from subnet.validator.database import ( + create_dump, + restore_dump, + set_version, +) + + +class VersionControl(BaseVersionControl): + def __init__(self, database): + super().__init__() + self.redis = Redis(database) + + def restart(self): + bt.logging.info(f"Restarting validator...") + python = sys.executable + os.execl(python, python, *sys.argv) + + async def upgrade_redis(self): + """ + Upgrade redis with the requested version or the latest one + Version has to follow the format major.minor.patch + """ + local_version = None + remote_version = None + is_upgrade = None + success = False + + try: + # Get latest version + remote_version = self.redis.get_latest_version() + bt.logging.info(f"[Redis] Remote version: {remote_version}") + + # Get the local version + active_version = await self.redis.get_version() + local_version = active_version or remote_version + bt.logging.info(f"[Redis] Local version: {local_version}") + + # Check if the subnet has to be upgraded + if local_version == remote_version: + if not active_version: + await set_version(remote_version, self.redis.database) + + bt.logging.success(f"[Redis] Already using {local_version}") + return (True, local_version, remote_version) + + self.must_restart = True + + # Dump the database + dump_name = f"redis-dump-{local_version}" + await create_dump(dump_name, self.redis.database) + bt.logging.info(f"[Redis] Dump {dump_name} created") + + local_version_num = int(local_version.replace(".", "")) + remote_version_num = int(remote_version.replace(".", "")) + + is_upgrade = remote_version_num > local_version_num + if is_upgrade: + # It is an upgrade so we have to use the new migrations directory + remove_dump_migrations() + + success = await self.redis.rollout(local_version, remote_version) + else: + # It is a downgrade so we have to use the old migrations directory + success = await self.redis.rollback(local_version, remote_version) + + if is_upgrade and not success: + bt.logging.info(f"[Redis] Rolling back to {local_version}...") + await self.redis.rollback(remote_version, local_version) + + except Exception as err: + bt.logging.error(f"[Redis] Upgrade failed: {err}") + if is_upgrade: + bt.logging.info(f"[Redis] Rolling back to {local_version}...") + success_rollback = await self.redis.rollback( + remote_version, local_version + ) + if not success_rollback: + dump_name = f"redis-dump-{local_version}" + await restore_dump(dump_name, self.redis.database) + bt.logging.info(f"[Redis] Dump {dump_name} restored") + + return (success, local_version, remote_version) + + def upgrade_validator(self): + current_version = None + remote_version = None + is_upgrade = None + success = False + + try: + # Get the remote version + remote_version = self.github.get_latest_version() + bt.logging.debug(f"[Subnet] Remote version: {remote_version}") + + # Get the local version + current_version = self.github.get_version() + bt.logging.debug(f"[Subnet] Local version: {current_version}") + + # Check if the subnet has to be upgraded + if current_version == remote_version: + bt.logging.success(f"[Subnet] Already using {current_version}") + return (True, current_version, remote_version) + + self.must_restart = True + + current_version_num = int(current_version.replace(".", "")) + remote_version_num = int(remote_version.replace(".", "")) + + is_upgrade = remote_version_num > current_version_num + if is_upgrade: + success = self.upgrade_subnet(remote_version) + else: + success = self.downgrade_subnet(remote_version) + + if is_upgrade and not success: + bt.logging.info(f"[Subnet] Rolling back to {current_version}...") + self.downgrade_subnet(current_version) + except Exception as err: + bt.logging.error(f"[Subnet] Upgrade failed: {err}") + if is_upgrade: + bt.logging.info(f"[Subnet] Rolling back to {current_version}...") + self.downgrade_subnet(current_version) + + return (success, current_version, remote_version) + + async def upgrade(self): + try: + # Flag to stop miner activity + self.upgrading = True + + # Create the migrations dump + create_dump_migrations() + + # Update Subnet + subnet_success, subnet_old_version, subnet_new_version = ( + self.upgrade_validator() + ) + if not subnet_success: + return self.must_restart + + # Upgrade redis + redis_success, _, _ = await self.upgrade_redis() + if ( + subnet_success + and subnet_old_version != subnet_new_version + and not redis_success + ): + bt.logging.info(f"[Subnet] Rolling back to {subnet_old_version}...") + self.downgrade_subnet(subnet_old_version) + + return self.must_restart + except Exception as ex: + bt.logging.error(f"Could not upgrade the validator: {ex}") + finally: + # Remove the migrations dump + remove_dump_migrations() + + # Unflag to resume miner activity + self.upgrading = False + self.must_restart = False + + return True diff --git a/subnet/version/github_controller.py b/subnet/version/github_controller.py new file mode 100644 index 00000000..d40d78f0 --- /dev/null +++ b/subnet/version/github_controller.py @@ -0,0 +1,65 @@ +import re +import os +import codecs +import requests +import subprocess +import bittensor as bt +from os import path + + +here = path.abspath(path.dirname(__file__)) + + +class Github: + def __init__(self, repo_owner="eclipsevortex", repo_name="SubVortexVC"): + self.repo_owner = repo_owner + self.repo_name = repo_name + + def get_version(self) -> str: + with codecs.open( + os.path.join(here, "../__init__.py"), encoding="utf-8" + ) as init_file: + version_match = re.search( + r"^__version__ = ['\"]([^'\"]*)['\"]", init_file.read(), re.M + ) + version_string = version_match.group(1) + return version_string + + def get_latest_version(self) -> str: + """ + Get the latest release on github + """ + url = f"https://api.github.com/repos/{self.repo_owner}/{self.repo_name}/releases/latest" + response = requests.get(url) + if response.status_code != 200: + return None + + latest_version = response.json()["tag_name"] + return latest_version[1:] + + def get_branch(self, tag="latest"): + """ + Get the expected branch + """ + if tag == "latest": + subprocess.run(["git", "checkout", "-B", "main"], check=True) + subprocess.run(["git", "pull"], check=True) + bt.logging.info(f"Successfully pulled source code for main branch'.") + else: + subprocess.run(["git", "checkout", f"tags/{tag}"], check=True) + bt.logging.info(f"Successfully pulled source code for tag '{tag}'.") + + def get_tag(self, tag): + """ + Get the expected tag + """ + # Stash if there is any local changes just in case + subprocess.run(["git", "stash"], check=True) + + # Fetch tags + subprocess.run(["git", "fetch", "--tags", "--force"], check=True) + bt.logging.info(f"Fetch tags.") + + # Checkout tags + subprocess.run(["git", "checkout", f"tags/{tag}"], check=True) + bt.logging.info(f"Successfully pulled source code for tag '{tag}'.") diff --git a/subnet/version/interpreter_controller.py b/subnet/version/interpreter_controller.py new file mode 100644 index 00000000..845de10b --- /dev/null +++ b/subnet/version/interpreter_controller.py @@ -0,0 +1,18 @@ +import subprocess +import bittensor as bt + + +class Interpreter: + def __init__(self): + pass + + def install_dependencies(self): + subprocess.run(["pip", "install", "-r", "requirements.txt"]) + bt.logging.info(f"Dependencies installed successfully") + + subprocess.run(["pip", "install", "-e", "."]) + bt.logging.info(f"Source installed successfully") + + def upgrade_dependencies(self): + subprocess.run(["pip", "install", "-r", "requirements.txt", "--upgrade"]) + bt.logging.info(f"Dependencies installed successfully") diff --git a/subnet/version/redis_controller.py b/subnet/version/redis_controller.py new file mode 100644 index 00000000..7f5c49e7 --- /dev/null +++ b/subnet/version/redis_controller.py @@ -0,0 +1,107 @@ +import bittensor as bt +from os import path + +from subnet.version.utils import get_migrations, get_migration_module +from subnet.validator.database import get_version as _get_version + +here = path.abspath(path.dirname(__file__)) + + +class Redis: + def __init__(self, database): + self.database = database + + async def get_version(self): + version = await _get_version(self.database) + return version + + def get_latest_version(self): + migration = get_migrations(True) + return migration[0][1] if len(migration) > 0 else None + + async def rollout(self, from_version: str, to_version: str): + """ + Rollout from from_version to to_version + """ + upper_version = int(to_version.replace(".", "")) + lower_version = int(from_version.replace(".", "")) + + # List all the migration to execute + migration_scripts = get_migrations() + migrations = [ + x + for x in migration_scripts + if x[0] > lower_version and x[0] <= upper_version + ] + migrations = sorted(migrations, key=lambda x: x[0]) + + version = None + try: + for migration in migrations: + version = migration[1] + + # Load the migration module + module = get_migration_module(migration[2]) + if not module: + bt.logging.error( + f"[Redis] Could not found the migration file {migration[2]}" + ) + return + + # Rollout the migration + await module.rollout(self.database) + + # Update the version in the database + new_version = await self.get_version() + if new_version: + bt.logging.success(f"[Redis] Rollout to {new_version} successful") + else: + bt.logging.success(f"[Redis] Rollout successful") + + return True + except Exception as err: + bt.logging.error(f"[Redis] Failed to upgrade to {version}: {err}") + + return False + + async def rollback(self, from_version: str, to_version: str = "0.0.0"): + upper_version = int(from_version.replace(".", "")) + lower_version = int(to_version.replace(".", "")) + + # List all the migration to execute + migration_scripts = get_migrations() + migrations = [ + x + for x in migration_scripts + if x[0] > lower_version and x[0] <= upper_version + ] + migrations = sorted(migrations, key=lambda x: x[0]) + + version = None + try: + for migration in migrations: + version = migration[1] + + # Load the migration module + module = get_migration_module(migration[2]) + if not module: + bt.logging.error( + f"[Redis] Could not found the migration file {migration[2]}" + ) + return + + # Rollback the migration + await module.rollback(self.database) + + # Update the version in the database + new_version = await self.get_version() + if new_version: + bt.logging.success(f"[Redis] Rollback to {new_version} successful") + else: + bt.logging.success(f"[Redis] Rollback successful") + + return True + except Exception as err: + bt.logging.error(f"[Redis] Failed to downgrade to {version}: {err}") + + return False diff --git a/subnet/version/utils.py b/subnet/version/utils.py new file mode 100644 index 00000000..da804af5 --- /dev/null +++ b/subnet/version/utils.py @@ -0,0 +1,102 @@ +import os +import re +import shutil +import importlib +import bittensor as bt +from os import path + +here = path.abspath(path.dirname(__file__)) + + +def extract_number(s): + """ + Extract the numbers (major, minor and patch) from the string version + """ + try: + match = re.search(r"(\d+)\.(\d+)\.(\d+)\.py", s) + if not match: + return None + + patch = int(match.group(3)) + minor = int(match.group(2)) + major = int(match.group(1)) + return (major, minor, patch) + except Exception as ex: + bt.logging.error(f"Could not extract the numbers from the string version: {ex}") + + return None + + +def get_migrations(force_new=False): + """ + List all the migrations available + """ + migrations = [] + + try: + source = os.path.join(here, "../../scripts/redis/previous_migrations") + if force_new or not os.path.exists(source): + source = os.path.join(here, "../../scripts/redis/migrations") + + # Load the migration scripts + files = os.listdir(source) + + # Get all the migration files + for file in files: + if not re.match(r"migration-[0-9]+\.[0-9]+\.[0-9]+.py", file): + continue + + version_details = extract_number(file) + if not version_details: + continue + + major, minor, patch = version_details + migrations.append( + (int(f"{major}{minor}{patch}"), f"{major}.{minor}.{patch}", file) + ) + + # Sort migration per version + migrations = sorted(migrations, key=lambda x: x[0], reverse=True) + + except Exception as ex: + bt.logging.error(f"Could not load the migrations: {ex}") + + return migrations + + +def create_dump_migrations(): + """ + Create a migrations dump + """ + source = os.path.join(here, "../../scripts/redis/migrations") + if not os.path.exists(source): + return + + target = os.path.join(here, "../../scripts/redis/previous_migrations") + if os.path.exists(target): + shutil.rmtree(target) + + # Dump the migrations directory + shutil.copytree(source, target) + + +def remove_dump_migrations(): + """ + Remove the migrations dump + """ + target = os.path.join(here, "../../scripts/redis/previous_migrations") + if not os.path.exists(target): + return + + shutil.rmtree(target) + + +def get_migration_module(migration: str): + file_path = f"scripts/redis/previous_migrations/{migration}" + if not os.path.exists(file_path): + file_path = f"scripts/redis/migrations/{migration}" + + spec = importlib.util.spec_from_file_location("migration_module", file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module diff --git a/subnet/version/version_control.py b/subnet/version/version_control.py new file mode 100644 index 00000000..33740dce --- /dev/null +++ b/subnet/version/version_control.py @@ -0,0 +1,93 @@ +import asyncio +import bittensor as bt +from os import path + +from subnet.version.github_controller import Github +from subnet.version.interpreter_controller import Interpreter +from subnet.version.redis_controller import Redis + + +class VersionControl: + def __init__(self): + self.github = Github() + self.interpreter = Interpreter() + self.redis = Redis() + + def upgrade_subnet(self): + try: + # Get the local version + current_version = self.github.get_version() + bt.logging.info(f"[Subnet] Current version: {current_version}") + + # Get the remote version + remote_version = self.github.get_latest_version() + bt.logging.info(f"[Subnet] Remote version: {remote_version}") + + # Check if the subnet has to be upgraded + if current_version == remote_version: + bt.logging.success("[Subnet] Already up to date") + return + + bt.logging.info("[Subnet] Upgrading...") + + # Pull the branch + self.github.get_branch() + + # Install dependencies + self.interpreter.upgrade_dependencies() + + bt.logging.success("[Subnet] Upgrade successful") + except Exception as err: + bt.logging.error(f"Failed to upgrade the subnet: {err}") + + async def upgrade_redis(self): + current_version = None + latest_version = None + new_version = None + + try: + # Get the local version + current_version = await self.redis.get_version() + bt.logging.info(f"[Redis] Current version: {current_version}") + + # Get latest version + latest_version = self.redis.get_latest_version() + bt.logging.info(f"[Redis] Latest version: {latest_version}") + + # Check if the subnet has to be upgraded + if current_version == latest_version: + bt.logging.success("[Redis] Already up to date") + return + + new_version = await self.redis.rollout(current_version, latest_version) + except Exception as err: + bt.logging.error(f"Failed to upgrade redis: {err}") + + if new_version != latest_version: + await self.redis.rollback(new_version, current_version) + bt.logging.success("[Redis] Rollback successful") + else: + bt.logging.success(f"[Redis] Upgrade to {latest_version} successful") + + def upgrade_subtensor(self): + try: + pass + except Exception as err: + bt.logging.error(f"Failed to upgrade subtensor: {err}") + + async def upgrade(self): + try: + # Upgrade subnet + self.upgrade_subnet() + + # Upgrade redis + await self.upgrade_redis() + + # Upgrade subtensor + # self.upgrade_subtensor() + except Exception as err: + bt.logging.error(f"Upgrade failed: {err}") + + +if __name__ == "__main__": + asyncio.run(VersionControl().upgrade()) diff --git a/tests/unit_tests/mocks/mock_interpreter.py b/tests/unit_tests/mocks/mock_interpreter.py new file mode 100644 index 00000000..65d53785 --- /dev/null +++ b/tests/unit_tests/mocks/mock_interpreter.py @@ -0,0 +1,8 @@ +def upgrade_depdencies_side_effect(*args, **kwargs): + if upgrade_depdencies_side_effect.called: + # Do nothing on subsequent calls + return + else: + # Raise an error on the first call + upgrade_depdencies_side_effect.called = True + raise ValueError("Simulated error") \ No newline at end of file diff --git a/tests/unit_tests/mocks/mock_redis.py b/tests/unit_tests/mocks/mock_redis.py index a053d1e6..a3283c5f 100644 --- a/tests/unit_tests/mocks/mock_redis.py +++ b/tests/unit_tests/mocks/mock_redis.py @@ -27,3 +27,13 @@ def mock_get_statistics(hoktkeys: List[str]): ) return mocked_redis + + +def rollout_side_effect(*args, **kwargs): + if rollout_side_effect.called: + # Do nothing on subsequent calls + return True + else: + # Raise an error on the first call + rollout_side_effect.called = True + raise ValueError("Simulated error") diff --git a/tests/unit_tests/subnet/miner/test_miner_version_control.py b/tests/unit_tests/subnet/miner/test_miner_version_control.py new file mode 100644 index 00000000..27ea02e0 --- /dev/null +++ b/tests/unit_tests/subnet/miner/test_miner_version_control.py @@ -0,0 +1,108 @@ +import unittest +from unittest.mock import patch, MagicMock, call + +from tests.unit_tests.mocks.mock_interpreter import upgrade_depdencies_side_effect + +from subnet.miner.version import VersionControl + + +class TestMinerVersionControl(unittest.TestCase): + @patch("subnet.shared.version.Interpreter") + @patch("subnet.shared.version.Github") + def test_no_new_version_available_when_upgradring_should_do_nothing( + self, mock_github, mock_interpreter + ): + # Arrange + mock_github_class = MagicMock() + mock_github_class.get_version.return_value = "2.0.0" + mock_github_class.get_latest_version.return_value = "2.0.0" + mock_github.return_value = mock_github_class + + mock_interpreter_class = MagicMock() + mock_interpreter.return_value = mock_interpreter_class + + vc = VersionControl() + + # Act + must_restart = vc.upgrade() + + # Assert + mock_github_class.get_tag.assert_not_called() + mock_interpreter_class.upgrade_dependencies.assert_not_called() + assert False == must_restart + + @patch("subnet.shared.version.Interpreter") + @patch("subnet.shared.version.Github") + def test_new_higher_version_available_when_upgradring_should_upgrade_the_miner( + self, mock_github, mock_interpreter + ): + # Arrange + mock_github_class = MagicMock() + mock_github_class.get_version.return_value = "2.0.0" + mock_github_class.get_latest_version.return_value = "2.1.0" + mock_github.return_value = mock_github_class + + mock_interpreter_class = MagicMock() + mock_interpreter.return_value = mock_interpreter_class + + vc = VersionControl() + + # Act + must_restart = vc.upgrade() + + # Assert + mock_github_class.get_tag.assert_called_with("v2.1.0") + mock_interpreter_class.upgrade_dependencies.assert_called_once() + assert True == must_restart + + @patch("subnet.shared.version.Interpreter") + @patch("subnet.shared.version.Github") + def test_upgrading_to_a_new_higher_version_when_failing_should_downgrade_to_the_old_current_version( + self, mock_github, mock_interpreter + ): + # Arrange + mock_github_class = MagicMock() + mock_github_class.get_version.return_value = "2.0.0" + mock_github_class.get_latest_version.return_value = "2.1.0" + mock_github.return_value = mock_github_class + + mock_interpreter_class = MagicMock() + upgrade_depdencies_side_effect.called = False + mock_interpreter_class.upgrade_dependencies.side_effect = ( + upgrade_depdencies_side_effect + ) + mock_interpreter.return_value = mock_interpreter_class + + vc = VersionControl() + + # Act + must_restart = vc.upgrade() + + # Assert + mock_github_class.get_tag.assert_has_calls([call("v2.1.0"), call("v2.0.0")]) + mock_interpreter_class.upgrade_dependencies.assert_has_calls([call(), call()]) + assert True == must_restart + + @patch("subnet.shared.version.Interpreter") + @patch("subnet.shared.version.Github") + def test_current_version_removed_when_upgradring_should_downgrade_the_miner( + self, mock_github, mock_interpreter + ): + # Arrange + mock_github_class = MagicMock() + mock_github_class.get_version.return_value = "2.1.0" + mock_github_class.get_latest_version.return_value = "2.0.0" + mock_github.return_value = mock_github_class + + mock_interpreter_class = MagicMock() + mock_interpreter.return_value = mock_interpreter_class + + vc = VersionControl() + + # Act + must_restart = vc.upgrade() + + # Assert + mock_github_class.get_tag.assert_called_with("v2.0.0") + mock_interpreter_class.upgrade_dependencies.assert_called_once() + assert True == must_restart diff --git a/tests/unit_tests/subnet/validator/test_validator_version_control.py b/tests/unit_tests/subnet/validator/test_validator_version_control.py new file mode 100644 index 00000000..65dd4355 --- /dev/null +++ b/tests/unit_tests/subnet/validator/test_validator_version_control.py @@ -0,0 +1,528 @@ +import aioredis +import unittest +from unittest.mock import patch, MagicMock, call, AsyncMock + +from tests.unit_tests.mocks.mock_interpreter import upgrade_depdencies_side_effect +from tests.unit_tests.mocks.mock_redis import rollout_side_effect + +from subnet.validator.version import VersionControl + + +class TestValidatorVersionControl(unittest.IsolatedAsyncioTestCase): + @patch("subnet.validator.version.remove_dump_migrations") + @patch("subnet.validator.version.create_dump_migrations") + @patch("subnet.validator.version.restore_dump") + @patch("subnet.validator.version.create_dump") + @patch("subnet.shared.version.Interpreter") + @patch("subnet.shared.version.Github") + @patch("subnet.validator.version.Redis") + async def test_no_new_version_available_when_upgradring_should_do_nothing( + self, + mock_redis, + mock_github, + mock_interpreter, + mock_create_dump, + mock_restore_dump, + more_create_dump_migrations, + more_remove_dump_migrations, + ): + # Arrange + mock_redis_class = MagicMock() + mock_redis_class.database = AsyncMock(aioredis.Redis) + mock_redis_class.get_version = AsyncMock(return_value="2.0.0") + mock_redis_class.get_latest_version.return_value = "2.0.0" + mock_redis.return_value = mock_redis_class + + mock_github_class = MagicMock() + mock_github_class.get_version.return_value = "2.0.0" + mock_github_class.get_latest_version.return_value = "2.0.0" + mock_github.return_value = mock_github_class + + mock_interpreter_class = MagicMock() + mock_interpreter.return_value = mock_interpreter_class + + vc = VersionControl(mock_redis.database) + + # Act + must_restart = await vc.upgrade() + + # Assert + mock_github_class.get_tag.assert_not_called() + mock_interpreter_class.upgrade_dependencies.assert_not_called() + mock_create_dump.assert_not_called() + mock_redis_class.rollout.assert_not_called() + mock_redis_class.rollback.assert_not_called() + mock_restore_dump.assert_not_called() + more_create_dump_migrations.assert_called_once() + more_remove_dump_migrations.assert_called_once() + assert False == must_restart + + @patch("subnet.validator.version.remove_dump_migrations") + @patch("subnet.validator.version.create_dump_migrations") + @patch("subnet.validator.version.restore_dump") + @patch("subnet.validator.version.create_dump") + @patch("subnet.shared.version.Interpreter") + @patch("subnet.shared.version.Github") + @patch("subnet.validator.version.Redis") + async def test_new_higher_validator_version_available_when_upgradring_should_upgrade_the_validator( + self, + mock_redis, + mock_github, + mock_interpreter, + mock_create_dump, + mock_restore_dump, + more_create_dump_migrations, + more_remove_dump_migrations, + ): + # Arrange + mock_redis_class = MagicMock() + mock_redis_class.database = AsyncMock(aioredis.Redis) + mock_redis_class.get_version = AsyncMock(return_value="2.0.0") + mock_redis_class.get_latest_version.return_value = "2.0.0" + mock_redis.return_value = mock_redis_class + + mock_github_class = MagicMock() + mock_github_class.get_version.return_value = "2.0.0" + mock_github_class.get_latest_version.return_value = "2.1.0" + mock_github.return_value = mock_github_class + + mock_interpreter_class = MagicMock() + mock_interpreter.return_value = mock_interpreter_class + + vc = VersionControl(mock_redis.database) + + # Act + must_restart = await vc.upgrade() + + # Assert + mock_github_class.get_tag.assert_called_with("v2.1.0") + mock_interpreter_class.upgrade_dependencies.assert_called_once() + mock_redis_class.rollout.assert_not_called() + mock_redis_class.rollback.assert_not_called() + mock_create_dump.assert_not_called() + mock_restore_dump.assert_not_called() + more_create_dump_migrations.assert_called_once() + more_remove_dump_migrations.assert_called_once() + assert True == must_restart + + @patch("subnet.validator.version.remove_dump_migrations") + @patch("subnet.validator.version.create_dump_migrations") + @patch("subnet.validator.version.restore_dump") + @patch("subnet.validator.version.create_dump") + @patch("subnet.shared.version.Interpreter") + @patch("subnet.shared.version.Github") + @patch("subnet.validator.version.Redis") + async def test_new_validator_higher_version_available_when_failing_upgrading_should_rollback_the_validator_to_keep_the_current_version( + self, + mock_redis, + mock_github, + mock_interpreter, + mock_create_dump, + mock_restore_dump, + more_create_dump_migrations, + more_remove_dump_migrations, + ): + # Arrange + mock_redis_class = MagicMock() + mock_redis_class.database = AsyncMock(aioredis.Redis) + mock_redis_class.get_version = AsyncMock(return_value="2.0.0") + mock_redis_class.get_latest_version.return_value = "2.0.0" + mock_redis.return_value = mock_redis_class + + mock_github_class = MagicMock() + mock_github_class.get_version.return_value = "2.0.0" + mock_github_class.get_latest_version.return_value = "2.1.0" + mock_github.return_value = mock_github_class + + mock_interpreter_class = MagicMock() + upgrade_depdencies_side_effect.called = False + mock_interpreter_class.upgrade_dependencies.side_effect = ( + upgrade_depdencies_side_effect + ) + mock_interpreter.return_value = mock_interpreter_class + + vc = VersionControl(mock_redis.database) + + # Act + must_restart = await vc.upgrade() + + # Assert + mock_github_class.get_tag.assert_has_calls([call("v2.1.0"), call("v2.0.0")]) + mock_interpreter_class.upgrade_dependencies.assert_has_calls([call(), call()]) + mock_redis_class.rollout.assert_not_called() + mock_redis_class.rollback.assert_not_called() + mock_create_dump.assert_not_called() + mock_restore_dump.assert_not_called() + more_create_dump_migrations.assert_called_once() + more_remove_dump_migrations.assert_called_once() + assert True == must_restart + + @patch("subnet.validator.version.remove_dump_migrations") + @patch("subnet.validator.version.create_dump_migrations") + @patch("subnet.validator.version.restore_dump") + @patch("subnet.validator.version.create_dump") + @patch("subnet.shared.version.Interpreter") + @patch("subnet.shared.version.Github") + @patch("subnet.validator.version.Redis") + async def test_current_validator_version_removed_when_upgradring_should_downgrade_the_validator( + self, + mock_redis, + mock_github, + mock_interpreter, + mock_create_dump, + mock_restore_dump, + more_create_dump_migrations, + more_remove_dump_migrations, + ): + # Arrange + mock_redis_class = MagicMock() + mock_redis_class.database = AsyncMock(aioredis.Redis) + mock_redis_class.get_version = AsyncMock(return_value="2.0.0") + mock_redis_class.get_latest_version.return_value = "2.0.0" + mock_redis.return_value = mock_redis_class + + mock_github_class = MagicMock() + mock_github_class.get_version.return_value = "2.1.0" + mock_github_class.get_latest_version.return_value = "2.0.0" + mock_github.return_value = mock_github_class + + mock_interpreter_class = MagicMock() + mock_interpreter.return_value = mock_interpreter_class + + vc = VersionControl(mock_redis.database) + + # Act + must_restart = await vc.upgrade() + + # Assert + mock_github_class.get_tag.assert_called_with("v2.0.0") + mock_interpreter_class.upgrade_dependencies.assert_called_once() + mock_redis_class.rollout.assert_not_called() + mock_redis_class.rollback.assert_not_called() + mock_create_dump.assert_not_called() + mock_restore_dump.assert_not_called() + more_create_dump_migrations.assert_called_once() + more_remove_dump_migrations.assert_called_once() + assert True == must_restart + + @patch("subnet.validator.version.remove_dump_migrations") + @patch("subnet.validator.version.create_dump_migrations") + @patch("subnet.validator.version.restore_dump") + @patch("subnet.validator.version.create_dump") + @patch("subnet.shared.version.Interpreter") + @patch("subnet.shared.version.Github") + @patch("subnet.validator.version.Redis") + async def test_new_higher_redis_version_available_when_upgradring_should_upgrade_redis( + self, + mock_redis, + mock_github, + mock_interpreter, + mock_create_dump, + mock_restore_dump, + more_create_dump_migrations, + more_remove_dump_migrations, + ): + # Arrange + mock_redis_class = MagicMock() + mock_redis_class.get_version = AsyncMock(return_value="2.0.0") + mock_redis_class.get_latest_version.return_value = "2.1.0" + mock_redis_class.rollout = AsyncMock() + mock_redis.return_value = mock_redis_class + + mock_github_class = MagicMock() + mock_github_class.get_version.return_value = "2.0.0" + mock_github_class.get_latest_version.return_value = "2.0.0" + mock_github.return_value = mock_github_class + + mock_interpreter_class = MagicMock() + mock_interpreter.return_value = mock_interpreter_class + + mock_create_dump.return_value = AsyncMock() + + vc = VersionControl(mock_redis.database) + + # Act + must_restart = await vc.upgrade() + + # Assert + mock_github_class.get_tag.assert_not_called() + mock_interpreter_class.upgrade_dependencies.assert_not_called() + mock_redis_class.rollout.assert_called_once_with("2.0.0", "2.1.0") + mock_redis_class.rollback.assert_not_called() + mock_create_dump.assert_called_once_with( + "redis-dump-2.0.0", mock_redis_class.database + ) + mock_restore_dump.assert_not_called() + more_create_dump_migrations.assert_called_once() + more_remove_dump_migrations.assert_has_calls([call(), call()]) + assert True == must_restart + + @patch("subnet.validator.version.remove_dump_migrations") + @patch("subnet.validator.version.create_dump_migrations") + @patch("subnet.validator.version.restore_dump") + @patch("subnet.validator.version.create_dump") + @patch("subnet.shared.version.Interpreter") + @patch("subnet.shared.version.Github") + @patch("subnet.validator.version.Redis") + async def test_new_higher_redis_version_available_when_failing_upgrading_should_rollback_redis_to_keep_the_current_version( + self, + mock_redis, + mock_github, + mock_interpreter, + mock_create_dump, + mock_restore_dump, + more_create_dump_migrations, + more_remove_dump_migrations, + ): + # Arrange + mock_redis_class = MagicMock() + mock_redis_class.get_version = AsyncMock(return_value="2.0.0") + mock_redis_class.get_latest_version.return_value = "2.1.0" + rollout_side_effect.called = False + mock_redis_class.rollout.side_effect = rollout_side_effect + mock_redis_class.rollback = AsyncMock() + mock_redis.return_value = mock_redis_class + + mock_github_class = MagicMock() + mock_github_class.get_version.return_value = "2.0.0" + mock_github_class.get_latest_version.return_value = "2.0.0" + mock_github.return_value = mock_github_class + + mock_interpreter_class = MagicMock() + mock_interpreter.return_value = mock_interpreter_class + + mock_create_dump.return_value = AsyncMock() + + vc = VersionControl(mock_redis.database) + + # Act + must_restart = await vc.upgrade() + + # Assert + mock_github_class.get_tag.assert_not_called() + mock_interpreter_class.upgrade_dependencies.assert_not_called() + mock_redis_class.rollout.assert_called_once_with("2.0.0", "2.1.0") + mock_redis_class.rollback.assert_called_once_with("2.1.0", "2.0.0") + mock_create_dump.assert_called_once_with( + "redis-dump-2.0.0", mock_redis_class.database + ) + mock_restore_dump.assert_not_called() + more_create_dump_migrations.assert_called_once() + more_remove_dump_migrations.assert_has_calls([call(), call()]) + assert True == must_restart + + @patch("subnet.validator.version.remove_dump_migrations") + @patch("subnet.validator.version.create_dump_migrations") + @patch("subnet.validator.version.restore_dump") + @patch("subnet.validator.version.create_dump") + @patch("subnet.shared.version.Interpreter") + @patch("subnet.shared.version.Github") + @patch("subnet.validator.version.Redis") + async def test_new_higher_redis_version_available_when_failing_upgrading_and_failling_rollbacking_should_restore_the_dump_to_keep_the_current_version( + self, + mock_redis, + mock_github, + mock_interpreter, + mock_create_dump, + mock_restore_dump, + more_create_dump_migrations, + more_remove_dump_migrations, + ): + # Arrange + mock_redis_class = MagicMock() + mock_redis_class.get_version = AsyncMock(return_value="2.0.0") + mock_redis_class.get_latest_version.return_value = "2.1.0" + rollout_side_effect.called = False + mock_redis_class.rollout.side_effect = rollout_side_effect + mock_redis_class.rollback = AsyncMock(return_value=False) + mock_redis.return_value = mock_redis_class + + mock_github_class = MagicMock() + mock_github_class.get_version.return_value = "2.0.0" + mock_github_class.get_latest_version.return_value = "2.0.0" + mock_github.return_value = mock_github_class + + mock_interpreter_class = MagicMock() + mock_interpreter.return_value = mock_interpreter_class + + mock_create_dump.return_value = AsyncMock() + + vc = VersionControl(mock_redis.database) + + # Act + must_restart = await vc.upgrade() + + # Assert + mock_github_class.get_tag.assert_not_called() + mock_interpreter_class.upgrade_dependencies.assert_not_called() + mock_redis_class.rollout.assert_called_once_with("2.0.0", "2.1.0") + mock_redis_class.rollback.assert_called_once_with("2.1.0", "2.0.0") + mock_create_dump.assert_called_once_with( + "redis-dump-2.0.0", mock_redis_class.database + ) + mock_restore_dump.assert_called_once_with( + "redis-dump-2.0.0", mock_redis_class.database + ) + more_create_dump_migrations.assert_called_once() + more_remove_dump_migrations.assert_has_calls([call(), call()]) + assert True == must_restart + + @patch("subnet.validator.version.remove_dump_migrations") + @patch("subnet.validator.version.create_dump_migrations") + @patch("subnet.validator.version.restore_dump") + @patch("subnet.validator.version.create_dump") + @patch("subnet.shared.version.Interpreter") + @patch("subnet.shared.version.Github") + @patch("subnet.validator.version.Redis") + async def test_current_redis_version_removed_when_upgradring_should_downgrade_redis( + self, + mock_redis, + mock_github, + mock_interpreter, + mock_create_dump, + mock_restore_dump, + more_create_dump_migrations, + more_remove_dump_migrations, + ): + # Arrange + mock_redis_class = MagicMock() + mock_redis_class.database = AsyncMock(aioredis.Redis) + mock_redis_class.get_version = AsyncMock(return_value="2.1.0") + mock_redis_class.get_latest_version.return_value = "2.0.0" + mock_redis_class.rollout = AsyncMock() + mock_redis_class.rollback = AsyncMock() + mock_redis.return_value = mock_redis_class + + mock_github_class = MagicMock() + mock_github_class.get_version.return_value = "2.0.0" + mock_github_class.get_latest_version.return_value = "2.0.0" + mock_github.return_value = mock_github_class + + mock_interpreter_class = MagicMock() + mock_interpreter.return_value = mock_interpreter_class + + mock_create_dump.return_value = AsyncMock() + + vc = VersionControl(mock_redis.database) + + # Act + must_restart = await vc.upgrade() + + # Assert + mock_github_class.get_tag.assert_not_called() + mock_interpreter_class.upgrade_dependencies.assert_not_called() + mock_redis_class.rollout.assert_not_called() + mock_redis_class.rollback.assert_called_once_with("2.1.0", "2.0.0") + mock_create_dump.assert_called_once_with( + "redis-dump-2.1.0", mock_redis_class.database + ) + mock_restore_dump.assert_not_called() + more_create_dump_migrations.assert_called_once() + more_remove_dump_migrations.assert_called_once() + assert True == must_restart + + @patch("subnet.validator.version.remove_dump_migrations") + @patch("subnet.validator.version.create_dump_migrations") + @patch("subnet.validator.version.restore_dump") + @patch("subnet.validator.version.create_dump") + @patch("subnet.shared.version.Interpreter") + @patch("subnet.shared.version.Github") + @patch("subnet.validator.version.Redis") + async def test_new_higher_miner_and_redis_version_available_when_upgradring_should_upgrade_both( + self, + mock_redis, + mock_github, + mock_interpreter, + mock_create_dump, + mock_restore_dump, + more_create_dump_migrations, + more_remove_dump_migrations, + ): + # Arrange + mock_redis_class = MagicMock() + mock_redis_class.get_version = AsyncMock(return_value="2.0.0") + mock_redis_class.get_latest_version.return_value = "2.1.0" + mock_redis_class.rollout = AsyncMock() + mock_redis.return_value = mock_redis_class + + mock_github_class = MagicMock() + mock_github_class.get_version.return_value = "2.0.0" + mock_github_class.get_latest_version.return_value = "2.1.0" + mock_github.return_value = mock_github_class + + mock_interpreter_class = MagicMock() + mock_interpreter.return_value = mock_interpreter_class + + mock_create_dump.return_value = AsyncMock() + + vc = VersionControl(mock_redis.database) + + # Act + must_restart = await vc.upgrade() + + # Assert + mock_github_class.get_tag.assert_called_with("v2.1.0") + mock_interpreter_class.upgrade_dependencies.assert_called_once() + mock_redis_class.rollout.assert_called_once_with("2.0.0", "2.1.0") + mock_create_dump.assert_called_once_with( + "redis-dump-2.0.0", mock_redis_class.database + ) + mock_restore_dump.assert_not_called() + more_create_dump_migrations.assert_called_once() + more_remove_dump_migrations.assert_has_calls([call(), call()]) + assert True == must_restart + + @patch("subnet.validator.version.remove_dump_migrations") + @patch("subnet.validator.version.create_dump_migrations") + @patch("subnet.validator.version.restore_dump") + @patch("subnet.validator.version.create_dump") + @patch("subnet.shared.version.Interpreter") + @patch("subnet.shared.version.Github") + @patch("subnet.validator.version.Redis") + async def test_new_higher_miner_and_redis_version_available_when_validator_uprade_succeed_and_redis_upgrade_failed_should_rollback_the_validator_version( + self, + mock_redis, + mock_github, + mock_interpreter, + mock_create_dump, + mock_restore_dump, + more_create_dump_migrations, + more_remove_dump_migrations, + ): + # Arrange + mock_redis_class = MagicMock() + mock_redis_class.get_version = AsyncMock(return_value="2.0.0") + mock_redis_class.get_latest_version.return_value = "2.1.0" + mock_redis_class.rollout = AsyncMock() + rollout_side_effect.called = False + mock_redis_class.rollout.side_effect = rollout_side_effect + mock_redis_class.rollback = AsyncMock() + mock_redis.return_value = mock_redis_class + + mock_github_class = MagicMock() + mock_github_class.get_version.return_value = "2.0.0" + mock_github_class.get_latest_version.return_value = "2.1.0" + mock_github.return_value = mock_github_class + + mock_interpreter_class = MagicMock() + mock_interpreter.return_value = mock_interpreter_class + + mock_create_dump.return_value = AsyncMock() + + vc = VersionControl(mock_redis.database) + + # Act + must_restart = await vc.upgrade() + + # Assert + mock_github_class.get_tag.assert_has_calls([call("v2.1.0"), call("v2.0.0")]) + mock_interpreter_class.upgrade_dependencies.assert_has_calls([call(), call()]) + mock_redis_class.rollout.assert_called_once_with("2.0.0", "2.1.0") + mock_redis_class.rollback.assert_called_once_with("2.1.0", "2.0.0") + mock_create_dump.assert_called_once_with( + "redis-dump-2.0.0", mock_redis_class.database + ) + mock_restore_dump.assert_not_called() + more_create_dump_migrations.assert_called_once() + more_remove_dump_migrations.assert_has_calls([call(), call()]) + assert True == must_restart diff --git a/tests/unit_tests/subnet/version/test_get_migrations.py b/tests/unit_tests/subnet/version/test_get_migrations.py new file mode 100644 index 00000000..c0471602 --- /dev/null +++ b/tests/unit_tests/subnet/version/test_get_migrations.py @@ -0,0 +1,56 @@ +import unittest +from unittest.mock import patch + +from subnet.version.utils import get_migrations + + +class TestUtilVersionControl(unittest.IsolatedAsyncioTestCase): + @patch("os.listdir") + async def test_no_migration_available_should_return_an_empty_list( + self, mock_listdir + ): + # Arrange + mock_listdir.return_value = [] + + # Act + result = get_migrations() + + # Assert + assert 0 == len(result) + + @patch("os.listdir") + async def test_migration_available_should_return_a_list_in_the_right_order( + self, mock_listdir + ): + # Arrange + mock_listdir.return_value = [ + "migration-2.1.0.py", + "migration-2.0.0.py", + "migration-2.1.1.py", + ] + + # Act + result = get_migrations() + + # Assert + assert 3 == len(result) + assert (211, "2.1.1", "migration-2.1.1.py") == result[0] + assert (210, "2.1.0", "migration-2.1.0.py") == result[1] + assert (200, "2.0.0", "migration-2.0.0.py") == result[2] + + @patch("os.listdir") + async def test_migration_available_when_few_does_match_the_expected_pattern_should_return_a_list_without_these_wrong_formatted_files( + self, mock_listdir + ): + # Arrange + mock_listdir.return_value = [ + "migration2.1.1.py", + "migrations-2.0.0.py", + "migration-210.py", + ] + + # Act + result = get_migrations() + + # Assert + assert 0 == len(result) From 232d74f3c1443eedc88d432aad3140ff109d8e8c Mon Sep 17 00:00:00 2001 From: eclipsevortex Date: Mon, 6 May 2024 17:11:17 +0100 Subject: [PATCH 3/5] isolate wandb --- neurons/validator.py | 7 +- subnet/validator/config.py | 18 --- subnet/validator/state.py | 267 +++++++++++++++++++------------------ 3 files changed, 137 insertions(+), 155 deletions(-) diff --git a/neurons/validator.py b/neurons/validator.py index 9c5b5860..e64f45f2 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -287,16 +287,13 @@ async def run_forward(): if self.config.neuron.verbose: bt.logging.debug(f"block at end of step: {self.prev_step_block}") bt.logging.debug(f"Step took {time.time() - start_epoch} seconds") + self.step += 1 except Exception as err: bt.logging.error("Error in training loop", str(err)) bt.logging.debug(print_exception(type(err), err, err.__traceback__)) - - if self.wandb is not None: - self.wandb.finish() - assert self.wandb.run is None - bt.logging.debug("Finishing wandb run") + finish_wandb() # After all we have to ensure subtensor connection is closed properly finally: diff --git a/subnet/validator/config.py b/subnet/validator/config.py index 7a884a58..12aee69b 100644 --- a/subnet/validator/config.py +++ b/subnet/validator/config.py @@ -134,12 +134,6 @@ def add_args(cls, parser): help="The default epoch length (how often we set weights, measured in 12 second blocks).", default=100, ) - parser.add_argument( - "--neuron.disable_log_rewards", - action="store_true", - help="Disable all reward logging, suppresses reward functions and their values from being logged to wandb.", - default=False, - ) parser.add_argument( "--neuron.subscription_logging_path", type=str, @@ -250,24 +244,12 @@ def add_args(cls, parser): help="Runs wandb in offline mode.", default=False, ) - parser.add_argument( - "--wandb.weights_step_length", - type=int, - help="How many steps before we log the weights.", - default=10, - ) parser.add_argument( "--wandb.run_step_length", type=int, help="How many steps before we rollover to a new run.", default=360, ) - parser.add_argument( - "--wandb.notes", - type=str, - help="Notes to add to the wandb run.", - default="", - ) # Mocks parser.add_argument( diff --git a/subnet/validator/state.py b/subnet/validator/state.py index b4b908ce..eac83fdf 100644 --- a/subnet/validator/state.py +++ b/subnet/validator/state.py @@ -181,7 +181,7 @@ def log_miners_table(self, miners: List[Miner], commit=False): data=data, ) - self.wandb.log({"02. Miners/miners": miners}, commit=commit) + wandb.run.log({"02. Miners/miners": miners}, commit=commit) bt.logging.trace(f"log_miners_table() {len(data)} miners") @@ -239,7 +239,7 @@ def log_score(self, name: str, uids: List[int], miners: List[Miner], commit=Fals data[str(uid)] = getattr(miner, property_name) # Create the graph - self.wandb.log({f"04. Scores/{name}_score": data}, commit=commit) + wandb.run.log({f"04. Scores/{name}_score": data}, commit=commit) bt.logging.trace(f"log_score() {name} {len(data)} scores") @@ -261,7 +261,7 @@ def log_moving_averaged_score( data[str(uid)] = score # Create the graph - self.wandb.log({"04. Scores/moving_averaged_score": data}, commit=commit) + wandb.run.log({"04. Scores/moving_averaged_score": data}, commit=commit) bt.logging.trace(f"log_moving_averaged_score() {len(data)} moving averaged scores") @@ -279,12 +279,12 @@ def log_completion_times(self, uids: List[int], miners: List[Miner], commit=Fals data[str(uid)] = miner.process_time or 0 # Create the graph - self.wandb.log({"05. Miscellaneous/completion_times": data}, commit=commit) + wandb.run.log({"05. Miscellaneous/completion_times": data}, commit=commit) bt.logging.trace(f"log_completion_times() {len(data)} completion times") def log_event(self, uids: List[int], step_length=None): - if self.config.wandb.off or self.wandb is None: + if self.config.wandb.off or wandb.run is None: return bt.logging.info("log_event()") @@ -295,11 +295,9 @@ def log_event(self, uids: List[int], step_length=None): # Add overview metrics best_miner = max(miners, key=lambda item: item.score) - self.wandb.log({"01. Overview/best_uid": best_miner.uid}, commit=False) + wandb.run.log({"01. Overview/best_uid": best_miner.uid}, commit=False) if step_length: - self.wandb.log( - {"01. Overview/step_process_time": step_length}, commit=False - ) + wandb.run.log({"01. Overview/step_process_time": step_length}, commit=False) # Add the miner table log_miners_table(self, miners) @@ -324,129 +322,131 @@ def log_event(self, uids: List[int], step_length=None): def init_wandb(self): """Starts a new wandb run.""" - tags = [ - self.wallet.hotkey.ss58_address, - THIS_VERSION, - str(THIS_SPEC_VERSION), - f"netuid_{self.metagraph.netuid}", - self.country, - ] - - if self.config.mock: - tags.append("mock") - if self.config.neuron.disable_set_weights: - tags.append("disable_set_weights") - if self.config.neuron.disable_log_rewards: - tags.append("disable_log_rewards") - - wandb_config = { - key: copy.deepcopy(self.config.get(key, None)) - for key in ("neuron", "reward", "netuid", "wandb") - } - wandb_config["neuron"].pop("full_path", None) - - # Ensure "subvortex-team" and "test-subvortex-team" are used with the right subnet UID - # If user provide its own project name we keep it - project_name = self.config.wandb.project_name - if self.config.netuid == MAIN_SUBNET_UID and project_name.endswith( - "subvortex-team" - ): - project_name = "subvortex-team" - elif self.config.netuid == TESTNET_SUBNET_UID and project_name.endswith( - "subvortex-team" - ): - project_name = "test-subvortex-team" - bt.logging.debug( - f"Wandb project {project_name} used for Subnet {self.config.netuid}" - ) + try: + tags = [ + self.wallet.hotkey.ss58_address, + THIS_VERSION, + str(THIS_SPEC_VERSION), + f"netuid_{self.metagraph.netuid}", + self.country, + ] + + if self.config.mock: + tags.append("mock") + if self.config.neuron.disable_set_weights: + tags.append("disable_set_weights") + + wandb_config = { + key: copy.deepcopy(self.config.get(key, None)) + for key in ("neuron", "reward", "netuid", "wandb") + } + wandb_config["neuron"].pop("full_path", None) + + # Ensure "subvortex-team" and "test-subvortex-team" are used with the right subnet UID + # If user provide its own project name we keep it + project_name = self.config.wandb.project_name + if self.config.netuid == MAIN_SUBNET_UID and project_name.endswith( + "subvortex-team" + ): + project_name = "subvortex-team" + elif self.config.netuid == TESTNET_SUBNET_UID and project_name.endswith( + "subvortex-team" + ): + project_name = "test-subvortex-team" + bt.logging.debug( + f"Wandb project {project_name} used for Subnet {self.config.netuid}" + ) - # Get the list of current runs for the validator - api = wandb.Api() - runs = api.runs( - f"{self.config.wandb.entity}/{project_name}", - order="-created_at", - filters={"display_name": {"$regex": f"^validator-{self.uid}"}}, - ) + # Get the list of current runs for the validator + api = wandb.Api() + runs = api.runs( + f"{self.config.wandb.entity}/{project_name}", + order="-created_at", + filters={"display_name": {"$regex": f"^validator-{self.uid}"}}, + ) - name = f"validator-{self.uid}-1" - if len(runs) > 0: - # Take the first run as it will be the most recent one - last_number = runs[0].name.split("-")[-1] - next_number = (int(last_number) % 10000) + 1 - name = f"validator-{self.uid}-{next_number}" - - # Create a new run - self.wandb = wandb.init( - anonymous="allow", - reinit=True, - project=project_name, - entity=self.config.wandb.entity, - config=wandb_config, - mode="offline" if self.config.wandb.offline else "online", - dir=self.config.neuron.full_path, - tags=tags, - notes=self.config.wandb.notes, - name=name, - ) + name = f"validator-{self.uid}-1" + if len(runs) > 0: + # Take the first run as it will be the most recent one + last_number = runs[0].name.split("-")[-1] + next_number = (int(last_number) % 10000) + 1 + name = f"validator-{self.uid}-{next_number}" + + # Create a new run + wandb.init( + anonymous="allow", + reinit=True, + project=project_name, + entity=self.config.wandb.entity, + config=wandb_config, + mode="offline" if self.config.wandb.offline else "online", + dir=self.config.neuron.full_path, + tags=tags, + name=name, + ) - bt.logging.debug(f"[Wandb] {len(runs)} run(s) exist") - - # Remove old runs - We keep only the new run - if len(runs) >= 1: - bt.logging.debug(f"[Wandb] Removing the {len(runs)} oldest run(s)") - for i in range(0, len(runs)): - run: public.Run = runs[i] - - # Remove remote run - run.delete(True) - bt.logging.debug(f"[Wandb] Run {run.name} removed remotely") - - # Remove local run - wandb_base = wandb.run.settings.wandb_dir - - if run.metadata is None: - pattern = r"run-\d{8}_\d{6}-" + re.escape(run.id) - - # matches = re.match(pattern, wandb_base) - matches = [ - subdir - for subdir in os.listdir(wandb_base) - if re.match(pattern, subdir) - ] - if len(matches) == 0: - continue - - run_local_path = f"{wandb_base}{matches[0]}" - bt.logging.debug("[Wandb] Local path computed") - else: - # Get the run started at time - startedAt = run.metadata["startedAt"] - - # Parse input datetime string into a datetime object - input_datetime = datetime.strptime(startedAt, "%Y-%m-%dT%H:%M:%S.%f") - - # Format the datetime object into the desired string format - output_datetime_str = input_datetime.strftime("%Y%m%d_%H%M%S") - - # Local path to the run files - run_local_path = f"{wandb_base}run-{output_datetime_str}-{run.id}" - bt.logging.debug("[Wandb] Local path retrieve from metadata") - - # Remove local run - if os.path.exists(run_local_path): - shutil.rmtree(run_local_path) - bt.logging.debug( - f"[Wandb] Run {run.name} removed locally {run_local_path}" - ) - else: - bt.logging.warning( - f"[Wandb] Run local directory {run_local_path} does not exist. Please check it has been removed." - ) - - bt.logging.success( - prefix="Started a new wandb run", - sufix=f" {self.wandb.name} ", - ) + bt.logging.debug(f"[Wandb] {len(runs)} run(s) exist") + + # Remove old runs - We keep only the new run + if len(runs) >= 1: + bt.logging.debug(f"[Wandb] Removing the {len(runs)} oldest run(s)") + for i in range(0, len(runs)): + run: public.Run = runs[i] + + # Remove remote run + run.delete(True) + bt.logging.debug(f"[Wandb] Run {run.name} removed remotely") + + # Remove local run + wandb_base = wandb.run.settings.wandb_dir + + if run.metadata is None: + pattern = r"run-\d{8}_\d{6}-" + re.escape(run.id) + + # matches = re.match(pattern, wandb_base) + matches = [ + subdir + for subdir in os.listdir(wandb_base) + if re.match(pattern, subdir) + ] + if len(matches) == 0: + continue + + run_local_path = f"{wandb_base}{matches[0]}" + bt.logging.debug("[Wandb] Local path computed") + else: + # Get the run started at time + startedAt = run.metadata["startedAt"] + + # Parse input datetime string into a datetime object + input_datetime = datetime.strptime( + startedAt, "%Y-%m-%dT%H:%M:%S.%f" + ) + + # Format the datetime object into the desired string format + output_datetime_str = input_datetime.strftime("%Y%m%d_%H%M%S") + + # Local path to the run files + run_local_path = f"{wandb_base}run-{output_datetime_str}-{run.id}" + bt.logging.debug("[Wandb] Local path retrieve from metadata") + + # Remove local run + if os.path.exists(run_local_path): + shutil.rmtree(run_local_path) + bt.logging.debug( + f"[Wandb] Run {run.name} removed locally {run_local_path}" + ) + else: + bt.logging.warning( + f"[Wandb] Run local directory {run_local_path} does not exist. Please check it has been removed." + ) + + bt.logging.success( + prefix="Started a new wandb run", + sufix=f" {wandb.run.name} ", + ) + except Exception as err: + bt.logging.warning(f"init_wandb() initialising wandb failed: {err}") def should_reinit_wandb(self): @@ -462,6 +462,9 @@ def finish_wandb(): """ Finish the current wandb run """ - bt.logging.debug("Finishing wandb run") - wandb.finish() - assert wandb.run is None + try: + bt.logging.debug("Finishing wandb run") + wandb.finish() + assert wandb.run is None + except Exception as err: + bt.logging.warning(f"finish_wandb() finishing wandb failed: {err}") From ffc94e808bf3d26a6ebd4e1f3483d38ad48af21a Mon Sep 17 00:00:00 2001 From: eclipsevortex Date: Mon, 6 May 2024 19:09:36 +0100 Subject: [PATCH 4/5] prepare new release --- CHANGELOG.md | 11 ++ VERSION | 2 +- .../release/release-2.2.3/RELEASE-2.2.3.md | 146 ++++++++++++++++++ subnet/__init__.py | 2 +- 4 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 scripts/release/release-2.2.3/RELEASE-2.2.3.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b1172af3..33ec5a5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 2.2.3 / 2024-05-06 + +## What's Changed +* Release/2.2.2 by @eclipsevortex in https://github.com/eclipsevortex/SubVortex/pull/36 +* Add unit tests for resync miners by @eclipsevortex in https://github.com/eclipsevortex/SubVortex/pull/38 +* implement auto upgrade by @eclipsevortex in https://github.com/eclipsevortex/SubVortex/pull/40 +* isolate wandb by @eclipsevortex in https://github.com/eclipsevortex/SubVortex/pull/41 + + +**Full Changelog**: https://github.com/eclipsevortex/SubVortex/compare/v2.2.2...v2.2.3 + ## 2.2.2 / 2024-04-25 **Full Changelog**: https://github.com/eclipsevortex/SubVortex/compare/v2.2.1...v2.2.2 diff --git a/VERSION b/VERSION index 7e541aec..6b4d1577 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2.2 \ No newline at end of file +2.2.3 \ No newline at end of file diff --git a/scripts/release/release-2.2.3/RELEASE-2.2.3.md b/scripts/release/release-2.2.3/RELEASE-2.2.3.md new file mode 100644 index 00000000..6a446171 --- /dev/null +++ b/scripts/release/release-2.2.3/RELEASE-2.2.3.md @@ -0,0 +1,146 @@ +This guide provides step-by-step instructions for the release 2.2.3. + +Previous Release: 2.2.2 + +
+ +--- + +- [Validator](#validators) + - [Rollout Process](#validator-rollout-process) + - [Rollback Process](#validator-rollback-process) +- [Miner](#miner) + - [Rollout Process](#miner-rollout-process) + - [Rollback Process](#miner-rollback-process) +- [Additional Resources](#additional-resources) +- [Troubleshooting](#troubleshooting) + +--- + +
+ +# Validator + +## Rollout Process + +1. **Upgrade Subnet**: Fetch the remote tags + ```bash + git fetch --tags --force + ``` + + Then, checkout the new release tag + ```bash + git checkout tags/v2.2.3 + ``` + + Finally, install the dependencies + + ```bash + pip install -r requirements.txt + pip install -e . + ``` + +2. **Restart validator**: Restart your validator to take the new version + + ```bash + pm2 restart validator-7 + ``` + +3. **Check logs**: Check the validator logs to see if you see some `New Block` + ```bash + pm2 logs validator-7 + ``` + +
+ +## Rollback Process + +If any issues arise during or after the rollout, follow these steps to perform a rollback: + +1. **Downgrade Subnet**: Checkout the previous release tag + ```bash + git checkout tags/v2.2.2 + ``` + + Then, install the dependencies + + ```bash + pip install -r requirements.txt + pip install -e . + ``` + +2. **Restart validator**: Restart your validator to take the new version + + ```bash + pm2 restart validator-7 + ``` + +3. **Check logs**: Check the validator logs to see if you see some `New Block` + ```bash + pm2 logs validator-7 + ``` + +
+ +# Miner + +## Rollout Process + +1. **Upgrade Subnet**: Fetch the remote tags + ```bash + git fetch --tags --force + ``` + + Then, checkout the new release tag + ```bash + git checkout tags/v2.2.3 + ``` + + Finally, install the dependencies + + ```bash + pip install -r requirements.txt + pip install -e . + ``` + +2. **Restart validator**: Restart your validator to take the new version + + ```bash + pm2 restart miner-7 + ``` + +3. **Check logs**: Check the validator logs to see if you see some `New Block` + ```bash + pm2 logs miner-7 + ``` + +## Rollback Process + +1. **Downgrade Subnet**: Checkout the previous release tag + ```bash + git checkout tags/v2.2.2 + ``` + + Then, install the dependencies + + ```bash + pip install -r requirements.txt + pip install -e . + ``` + +2. **Restart validator**: Restart your validator to take the new version + + ```bash + pm2 restart miner-7 + ``` + +3. **Check logs**: Check the validator logs to see if you see some `New Block` + ```bash + pm2 logs miner-7 + ``` + +
+ +# Additional Resources + +For any further assistance or inquiries, please contact [**SubVortex Team**](https://discord.com/channels/799672011265015819/1215311984799653918) diff --git a/subnet/__init__.py b/subnet/__init__.py index 3cf53e29..5deaf90e 100644 --- a/subnet/__init__.py +++ b/subnet/__init__.py @@ -50,7 +50,7 @@ def __lt__(self, other): ) -__version__ = "2.2.2" +__version__ = "2.2.3" version = SubnetVersion.from_string(__version__) __spec_version__ = version.to_spec_version() From 22ea24ab01602ea4ab4ec3d211743b3c0731b65e Mon Sep 17 00:00:00 2001 From: eclipsevortex Date: Mon, 6 May 2024 19:13:29 +0100 Subject: [PATCH 5/5] fix last details --- README.md | 2 + neurons/validator.py | 25 ++-- scripts/redis/README.md | 6 +- scripts/redis/utils/redis_dump.py | 2 +- .../release/release-2.2.3/RELEASE-2.2.3.md | 117 ++++++++++++++++-- scripts/setup_and_run.sh | 6 +- scripts/subnet/README.md | 4 +- subnet/miner/run.py | 9 +- subnet/shared/utils.py | 9 +- subnet/validator/config.py | 6 + subnet/validator/version.py | 14 ++- subnet/version/github_controller.py | 20 +-- subnet/version/redis_controller.py | 51 ++++---- subnet/version/utils.py | 8 +- .../test_validator_version_control.py | 50 +++++--- .../subnet/version/test_get_latest_version.py | 99 +++++++++++++++ .../subnet/version/test_get_migrations.py | 98 ++++++++++++++- 17 files changed, 424 insertions(+), 102 deletions(-) create mode 100644 tests/unit_tests/subnet/version/test_get_latest_version.py diff --git a/README.md b/README.md index 973250bb..e668cc08 100644 --- a/README.md +++ b/README.md @@ -376,6 +376,8 @@ pm2 start neurons/validator.py \ > NOTE: to access the wandb UI to get statistics about the miners, you can click on this [link](https://wandb.ai/eclipsevortext/subvortex-team) and choose the validator run you want. +> NOTE: by default the dumps created by the auto-update will be stored in /etc/redis. If you want to change the location, please use `--database.redis_dump_path`. + ## Releases - [Release-2.2.0](./scripts/release/release-2.2.0/RELEASE-2.2.0.md) diff --git a/neurons/validator.py b/neurons/validator.py index e64f45f2..47942165 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -193,7 +193,8 @@ async def run(self): bt.logging.info("run()") # Initi versioin control - self.version_control = VersionControl(self.database) + dump_path = self.config.database.redis_dump_path + self.version_control = VersionControl(self.database, dump_path) # Init miners self.miners = await get_all_miners(self) @@ -208,6 +209,17 @@ async def run(self): try: while 1: + # Start the upgrade process every 10 minutes + if should_upgrade(self.config.auto_update, self.last_upgrade_check): + bt.logging.debug("Checking upgrade") + must_restart = await self.version_control.upgrade() + if must_restart: + finish_wandb() + self.version_control.restart() + return + + self.last_upgrade_check = time.time() + start_epoch = time.time() await resync_metagraph_and_miners(self) @@ -266,17 +278,6 @@ async def run_forward(): prev_set_weights_block = get_current_block(self.subtensor) save_state(self) - # Start the upgrade process every 10 minutes - if should_upgrade(self.config.auto_update, self.last_upgrade_check): - bt.logging.debug("Checking upgrade") - must_restart = await self.version_control.upgrade() - if must_restart: - finish_wandb() - self.version_control.restart() - return - - self.last_upgrade_check = time.time() - # Rollover wandb to a new run. if should_reinit_wandb(self): bt.logging.info("Reinitializing wandb") diff --git a/scripts/redis/README.md b/scripts/redis/README.md index e401c9cf..130f9157 100644 --- a/scripts/redis/README.md +++ b/scripts/redis/README.md @@ -296,7 +296,7 @@ To create a Redis dump manually, you can use the python script `redis_dump.py`. For example, if you want to create the dump in the `subVortex` directory, you can run ``` -python3 ./scripts/redis/utils/redis_dump.py --run-type create +python3 ./scripts/redis/utils/redis_dump.py --run-type create --dump-path redis-dump-2.0.0.json ``` If you want to create the dump in another location and/or name, you can use the argument `--dump-path` @@ -312,11 +312,11 @@ To restore a Redis dump manually, you can use the python script `redis_dump.py`. For example, if you want to create in `subVortex` directory, you can run ``` -python3 ./scripts/redis/utils/redis_dump.py --run-type restore +python3 ./scripts/redis/utils/redis_dump.py --run-type restore --dump-path redis-dump-2.0.0.json ``` If you want to restore a dump in another location, you can use the argument `--dump-path` ``` -python3 ./scripts/redis/utils/redis_dump.py --run-type restore --dump-path /tmp/redis +python3 ./scripts/redis/utils/redis_dump.py --run-type restore --dump-path /tmp/redis/redis-dump-2.0.0.json ``` diff --git a/scripts/redis/utils/redis_dump.py b/scripts/redis/utils/redis_dump.py index dc7da88f..71a2d3ad 100644 --- a/scripts/redis/utils/redis_dump.py +++ b/scripts/redis/utils/redis_dump.py @@ -71,7 +71,7 @@ async def main(args): parser.add_argument( "--dump-path", type=str, - default="", + default="/tmp/redis", help="Dump file (with path) to create or restore", ) parser.add_argument( diff --git a/scripts/release/release-2.2.3/RELEASE-2.2.3.md b/scripts/release/release-2.2.3/RELEASE-2.2.3.md index 6a446171..8142681f 100644 --- a/scripts/release/release-2.2.3/RELEASE-2.2.3.md +++ b/scripts/release/release-2.2.3/RELEASE-2.2.3.md @@ -24,11 +24,13 @@ Previous Release: 2.2.2 ## Rollout Process 1. **Upgrade Subnet**: Fetch the remote tags + ```bash git fetch --tags --force ``` Then, checkout the new release tag + ```bash git checkout tags/v2.2.3 ``` @@ -40,13 +42,38 @@ Previous Release: 2.2.2 pip install -e . ``` -2. **Restart validator**: Restart your validator to take the new version +2. **Delete validator**: Remove your validator ```bash - pm2 restart validator-7 + pm2 delete validator-7 ``` -3. **Check logs**: Check the validator logs to see if you see some `New Block` + Use `pm2 show validator-7` to get the list of arguments you were using to be able to restore them in the step 3. + +3. **Start validator in auto-upgrade mode**: Start the validator by running in **Subvortex** + + ```bash + pm2 start neurons/validator.py -f \ + --name validator-7 \ + --interpreter python3 -- \ + --netuid 7 \ + --wallet.name $WALLET_NAME \ + --wallet.hotkey $HOTKEY_NAME \ + --subtensor.chain_endpoint ws://$IP:9944 \ + --logging.debug \ + --auto-update + ``` + + Replace **$WALLET_NAME**, **$HOTKEY_NAME** and **$IP** by the expected value. + If you had other arguments, please add them! + + > IMPORTANT
+ > Do not forget to provide the `--auto-update` argument. + + > IMPORTANT
+ > Use wandb without overriding the default value, as it will enable the Subvortex team to monitor the version of the validators and take action if necessary. + +4. **Check logs**: Check the validator logs to see if you see some `New Block` ```bash pm2 logs validator-7 ``` @@ -58,6 +85,7 @@ Previous Release: 2.2.2 If any issues arise during or after the rollout, follow these steps to perform a rollback: 1. **Downgrade Subnet**: Checkout the previous release tag + ```bash git checkout tags/v2.2.2 ``` @@ -69,13 +97,34 @@ If any issues arise during or after the rollout, follow these steps to perform a pip install -e . ``` -2. **Restart validator**: Restart your validator to take the new version +2. **Delete validator**: Remove your validator ```bash - pm2 restart validator-7 + pm2 delete validator-7 ``` -3. **Check logs**: Check the validator logs to see if you see some `New Block` + Use `pm2 show validator-7` to get the list of arguments you were using to be able to restore them in the step 3. + +3. **Start validator**: Start the validator by running in **Subvortex** + + ```bash + pm2 start neurons/validator.py -f \ + --name validator-7 \ + --interpreter python3 -- \ + --netuid 7 \ + --wallet.name $WALLET_NAME \ + --wallet.hotkey $HOTKEY_NAME \ + --subtensor.chain_endpoint ws://$IP:9944 \ + --logging.debug + ``` + + Replace **$WALLET_NAME**, **$HOTKEY_NAME** and **$IP** by the expected value. + If you had other arguments, please add them! + + > IMPORTANT
+ > Do not forget to remove the `--auto-update` argument. + +4. **Check logs**: Check the validator logs to see if you see some `New Block` ```bash pm2 logs validator-7 ``` @@ -87,11 +136,13 @@ If any issues arise during or after the rollout, follow these steps to perform a ## Rollout Process 1. **Upgrade Subnet**: Fetch the remote tags + ```bash git fetch --tags --force ``` Then, checkout the new release tag + ```bash git checkout tags/v2.2.3 ``` @@ -103,13 +154,34 @@ If any issues arise during or after the rollout, follow these steps to perform a pip install -e . ``` -2. **Restart validator**: Restart your validator to take the new version +2. **Delete miner**: Remove your miner ```bash - pm2 restart miner-7 + pm2 delete miner-7 ``` -3. **Check logs**: Check the validator logs to see if you see some `New Block` + Use `pm2 show miner-7` to get the list of arguments you were using to be able to restore them in the step 3. + +3. **Start validator in auto-upgrade mode**: Start the miner by running in **Subvortex** + + ```bash + pm2 start neurons/miner.py -f \ + --name miner-7 \ + --interpreter python3 -- \ + --netuid 7 \ + --wallet.name $WALLET_NAME \ + --wallet.hotkey $HOTKEY_NAME \ + --logging.debug \ + --auto-update + ``` + + Replace **$WALLET_NAME**, **$HOTKEY_NAME** and **$IP** by the expected value. + If you had other arguments, please add them! + + > IMPORTANT
+ > Do not forget to provide the `--auto-update` argument. + +4. **Check logs**: Check the miner logs to see if you see some `New Block` ```bash pm2 logs miner-7 ``` @@ -117,6 +189,7 @@ If any issues arise during or after the rollout, follow these steps to perform a ## Rollback Process 1. **Downgrade Subnet**: Checkout the previous release tag + ```bash git checkout tags/v2.2.2 ``` @@ -128,13 +201,33 @@ If any issues arise during or after the rollout, follow these steps to perform a pip install -e . ``` -2. **Restart validator**: Restart your validator to take the new version +2. **Delete miner**: Remove your miner + + ```bash + pm2 delete miner-7 + ``` + + Use `pm2 show miner-7` to get the list of arguments you were using to be able to restore them in the step 3. + +3. **Start miner**: Start the miner by running in **Subvortex** ```bash - pm2 restart miner-7 + pm2 start neurons/miner.py -f \ + --name miner-7 \ + --interpreter python3 -- \ + --netuid 7 \ + --wallet.name $WALLET_NAME \ + --wallet.hotkey $HOTKEY_NAME \ + --logging.debug ``` -3. **Check logs**: Check the validator logs to see if you see some `New Block` + Replace **$WALLET_NAME**, **$HOTKEY_NAME** and **$IP** by the expected value. + If you had other arguments, please add them! + + > IMPORTANT
+ > Do not forget to remove the `--auto-update` argument + +4. **Check logs**: Check the miner logs to see if you see some `New Block` ```bash pm2 logs miner-7 ``` diff --git a/scripts/setup_and_run.sh b/scripts/setup_and_run.sh index 872e80be..3c62e12d 100755 --- a/scripts/setup_and_run.sh +++ b/scripts/setup_and_run.sh @@ -212,7 +212,8 @@ if [[ "$TYPE" == "miner" ]]; then --subtensor.network local \ --wallet.name $WALLET_NAME \ --wallet.hotkey $HOTKEY_NAME \ - --logging.debug + --logging.debug \ + --auto-update fi # Run validator @@ -239,6 +240,7 @@ if [[ "$TYPE" == "validator" ]]; then --netuid $NETUID \ --wallet.name $WALLET_NAME \ --wallet.hotkey $HOTKEY_NAME \ - --logging.debug \ + --logging.debug \ + --auto-update \ $OPTIONS fi diff --git a/scripts/subnet/README.md b/scripts/subnet/README.md index 53697522..565067dc 100644 --- a/scripts/subnet/README.md +++ b/scripts/subnet/README.md @@ -72,7 +72,7 @@ To uprade the Subnet manually, you can use the python script `subnet_upgrade.py` For example, if you are on tag v2.2.2 and want to migrate to the tag v2.2.3, you can run in `SubVortex` ``` -python3 ./scripts/subnet/subnet_upgrade.py --tag v2.2.3 +python3 ./scripts/subnet/utils/subnet_upgrade.py --tag v2.2.3 ``` ## Downgrade @@ -82,5 +82,5 @@ To downgrade the Subnet manually, you can use the python script `subnet_upgrade. For example, if you are on tag v2.2.3 and want to downgrade to the tag v2.2.2, you can run in `SubVortex` ``` -python3 ./scripts/subnet/subnet_upgrade.py --tag v2.2.2 +python3 ./scripts/subnet/utils/subnet_upgrade.py --tag v2.2.2 ``` diff --git a/subnet/miner/run.py b/subnet/miner/run.py index ad444ac4..ac3f1d5a 100644 --- a/subnet/miner/run.py +++ b/subnet/miner/run.py @@ -7,6 +7,7 @@ from subnet.miner.version import VersionControl + def run(self): """ Initiates and manages the main loop for the miner on the Bittensor network. @@ -41,7 +42,7 @@ def run(self): netuid = self.config.netuid - version_control = VersionControl() + self.version_control = VersionControl() # Keep a track of last upgrade check self.last_upgrade_check = 0 @@ -63,11 +64,11 @@ def handler(obj, update_nr, subscription_id): if should_upgrade(self.config.auto_update, self.last_upgrade_check): bt.logging.debug("Checking upgrade") - must_restart = version_control.upgrade() + must_restart = self.version_control.upgrade() if must_restart: self.version_control.restart() return - + self.last_upgrade_check = time.time() bt.logging.debug( @@ -77,4 +78,4 @@ def handler(obj, update_nr, subscription_id): if self.should_exit: return True - block_handler_substrate.subscribe_block_headers(handler) \ No newline at end of file + block_handler_substrate.subscribe_block_headers(handler) diff --git a/subnet/shared/utils.py b/subnet/shared/utils.py index 0da04823..0f5007a0 100644 --- a/subnet/shared/utils.py +++ b/subnet/shared/utils.py @@ -20,6 +20,10 @@ import subprocess import bittensor as bt +# Check if there is an update every 5 minutes +# Github Rate Limit 60 requests per hour / per ip (Reference: https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28) +CHECK_UPDATE_FREQUENCY = 5 * 60 + def get_redis_password( redis_password: str = None, redis_conf: str = "/etc/redis/redis.conf" @@ -48,8 +52,7 @@ def get_redis_password( def should_upgrade(auto_update: bool, last_upgrade_check: float): """ - True if it is sime to upgrade, false otherwise - For now, upgrading evering 60 seconds + True if it is time to upgrade, false otherwise """ time_since_last_update = time.time() - last_upgrade_check - return time_since_last_update >= 60 and auto_update + return time_since_last_update >= CHECK_UPDATE_FREQUENCY and auto_update diff --git a/subnet/validator/config.py b/subnet/validator/config.py index 12aee69b..c5c2e335 100644 --- a/subnet/validator/config.py +++ b/subnet/validator/config.py @@ -213,6 +213,12 @@ def add_args(cls, parser): help="Redis configuration path.", default="/etc/redis/redis.conf", ) + parser.add_argument( + "--database.redis_dump_path", + type=str, + help="Redis directory where to store dumps.", + default="/etc/redis/", + ) # Auto update parser.add_argument( diff --git a/subnet/validator/version.py b/subnet/validator/version.py index 0c9a974b..9dc12b4e 100644 --- a/subnet/validator/version.py +++ b/subnet/validator/version.py @@ -11,11 +11,13 @@ set_version, ) +LAST_VERSION_BEFORE_AUTO_UPDATE = "2.2.0" + class VersionControl(BaseVersionControl): - def __init__(self, database): + def __init__(self, database, dump_path: str): super().__init__() - self.redis = Redis(database) + self.redis = Redis(database, dump_path) def restart(self): bt.logging.info(f"Restarting validator...") @@ -39,7 +41,7 @@ async def upgrade_redis(self): # Get the local version active_version = await self.redis.get_version() - local_version = active_version or remote_version + local_version = active_version or LAST_VERSION_BEFORE_AUTO_UPDATE bt.logging.info(f"[Redis] Local version: {local_version}") # Check if the subnet has to be upgraded @@ -53,7 +55,8 @@ async def upgrade_redis(self): self.must_restart = True # Dump the database - dump_name = f"redis-dump-{local_version}" + dump_path = self.redis.dump_path + dump_name = os.path.join(dump_path, f"redis-dump-{local_version}") await create_dump(dump_name, self.redis.database) bt.logging.info(f"[Redis] Dump {dump_name} created") @@ -82,7 +85,8 @@ async def upgrade_redis(self): remote_version, local_version ) if not success_rollback: - dump_name = f"redis-dump-{local_version}" + dump_path = self.redis.dump_path + dump_name = os.path.join(dump_path, f"redis-dump-{local_version}") await restore_dump(dump_name, self.redis.database) bt.logging.info(f"[Redis] Dump {dump_name} restored") diff --git a/subnet/version/github_controller.py b/subnet/version/github_controller.py index d40d78f0..ca6573f2 100644 --- a/subnet/version/github_controller.py +++ b/subnet/version/github_controller.py @@ -11,9 +11,10 @@ class Github: - def __init__(self, repo_owner="eclipsevortex", repo_name="SubVortexVC"): + def __init__(self, repo_owner="eclipsevortex", repo_name="SubVortex"): self.repo_owner = repo_owner self.repo_name = repo_name + self.latest_version = None def get_version(self) -> str: with codecs.open( @@ -28,14 +29,19 @@ def get_version(self) -> str: def get_latest_version(self) -> str: """ Get the latest release on github + Return the cached value if any errors """ - url = f"https://api.github.com/repos/{self.repo_owner}/{self.repo_name}/releases/latest" - response = requests.get(url) - if response.status_code != 200: - return None + try: + url = f"https://api.github.com/repos/{self.repo_owner}/{self.repo_name}/releases/latest" + response = requests.get(url) + if response.status_code != 200: + return self.latest_version - latest_version = response.json()["tag_name"] - return latest_version[1:] + latest_version = response.json()["tag_name"] + self.latest_version = latest_version[1:] + return self.latest_version + except Exception: + return self.latest_version def get_branch(self, tag="latest"): """ diff --git a/subnet/version/redis_controller.py b/subnet/version/redis_controller.py index 7f5c49e7..8bbdd880 100644 --- a/subnet/version/redis_controller.py +++ b/subnet/version/redis_controller.py @@ -8,15 +8,16 @@ class Redis: - def __init__(self, database): + def __init__(self, database, dump_path: str): self.database = database + self.dump_path = dump_path async def get_version(self): version = await _get_version(self.database) return version def get_latest_version(self): - migration = get_migrations(True) + migration = get_migrations(force_new=True, reverse=True) return migration[0][1] if len(migration) > 0 else None async def rollout(self, from_version: str, to_version: str): @@ -27,13 +28,9 @@ async def rollout(self, from_version: str, to_version: str): lower_version = int(from_version.replace(".", "")) # List all the migration to execute - migration_scripts = get_migrations() - migrations = [ - x - for x in migration_scripts - if x[0] > lower_version and x[0] <= upper_version - ] - migrations = sorted(migrations, key=lambda x: x[0]) + migrations = get_migrations( + filter_lambda=lambda x: x[0] > lower_version and x[0] <= upper_version + ) version = None try: @@ -51,14 +48,13 @@ async def rollout(self, from_version: str, to_version: str): # Rollout the migration await module.rollout(self.database) - # Update the version in the database - new_version = await self.get_version() - if new_version: - bt.logging.success(f"[Redis] Rollout to {new_version} successful") - else: - bt.logging.success(f"[Redis] Rollout successful") + # Log to keep track + bt.logging.debug(f"[Redis] Rollout to {version} successful") + + # Update the version in the database + bt.logging.success(f"[Redis] Rollout to {to_version} successful") - return True + return True except Exception as err: bt.logging.error(f"[Redis] Failed to upgrade to {version}: {err}") @@ -69,13 +65,10 @@ async def rollback(self, from_version: str, to_version: str = "0.0.0"): lower_version = int(to_version.replace(".", "")) # List all the migration to execute - migration_scripts = get_migrations() - migrations = [ - x - for x in migration_scripts - if x[0] > lower_version and x[0] <= upper_version - ] - migrations = sorted(migrations, key=lambda x: x[0]) + migrations = get_migrations( + reverse=True, + filter_lambda=lambda x: x[0] > lower_version and x[0] <= upper_version, + ) version = None try: @@ -93,12 +86,14 @@ async def rollback(self, from_version: str, to_version: str = "0.0.0"): # Rollback the migration await module.rollback(self.database) - # Update the version in the database - new_version = await self.get_version() - if new_version: - bt.logging.success(f"[Redis] Rollback to {new_version} successful") + # Log to keep track + if version: + bt.logging.debug(f"[Redis] Rollback from {version} successful") else: - bt.logging.success(f"[Redis] Rollback successful") + bt.logging.debug("[Redis] Rollback successful") + + # Update the version in the database + bt.logging.success(f"[Redis] Rollback to {to_version} successful") return True except Exception as err: diff --git a/subnet/version/utils.py b/subnet/version/utils.py index da804af5..6b46fa50 100644 --- a/subnet/version/utils.py +++ b/subnet/version/utils.py @@ -27,7 +27,7 @@ def extract_number(s): return None -def get_migrations(force_new=False): +def get_migrations(force_new=False, reverse=False, filter_lambda = None): """ List all the migrations available """ @@ -55,8 +55,12 @@ def get_migrations(force_new=False): (int(f"{major}{minor}{patch}"), f"{major}.{minor}.{patch}", file) ) + # Filter the migrations + if filter_lambda: + migrations = filter(filter_lambda, migrations) + # Sort migration per version - migrations = sorted(migrations, key=lambda x: x[0], reverse=True) + migrations = sorted(migrations, key=lambda x: x[0], reverse=reverse) except Exception as ex: bt.logging.error(f"Could not load the migrations: {ex}") diff --git a/tests/unit_tests/subnet/validator/test_validator_version_control.py b/tests/unit_tests/subnet/validator/test_validator_version_control.py index 65dd4355..fd75d5ea 100644 --- a/tests/unit_tests/subnet/validator/test_validator_version_control.py +++ b/tests/unit_tests/subnet/validator/test_validator_version_control.py @@ -31,6 +31,7 @@ async def test_no_new_version_available_when_upgradring_should_do_nothing( mock_redis_class.database = AsyncMock(aioredis.Redis) mock_redis_class.get_version = AsyncMock(return_value="2.0.0") mock_redis_class.get_latest_version.return_value = "2.0.0" + mock_redis_class.dump_path = "/etc/redis" mock_redis.return_value = mock_redis_class mock_github_class = MagicMock() @@ -41,7 +42,9 @@ async def test_no_new_version_available_when_upgradring_should_do_nothing( mock_interpreter_class = MagicMock() mock_interpreter.return_value = mock_interpreter_class - vc = VersionControl(mock_redis.database) + vc = VersionControl( + mock_redis.database, mock_redis_class.dump_path + ) # Act must_restart = await vc.upgrade() @@ -79,6 +82,7 @@ async def test_new_higher_validator_version_available_when_upgradring_should_upg mock_redis_class.database = AsyncMock(aioredis.Redis) mock_redis_class.get_version = AsyncMock(return_value="2.0.0") mock_redis_class.get_latest_version.return_value = "2.0.0" + mock_redis_class.dump_path = "/etc/redis" mock_redis.return_value = mock_redis_class mock_github_class = MagicMock() @@ -89,7 +93,9 @@ async def test_new_higher_validator_version_available_when_upgradring_should_upg mock_interpreter_class = MagicMock() mock_interpreter.return_value = mock_interpreter_class - vc = VersionControl(mock_redis.database) + vc = VersionControl( + mock_redis.database, mock_redis_class.dump_path + ) # Act must_restart = await vc.upgrade() @@ -127,6 +133,7 @@ async def test_new_validator_higher_version_available_when_failing_upgrading_sho mock_redis_class.database = AsyncMock(aioredis.Redis) mock_redis_class.get_version = AsyncMock(return_value="2.0.0") mock_redis_class.get_latest_version.return_value = "2.0.0" + mock_redis_class.dump_path = "/etc/redis" mock_redis.return_value = mock_redis_class mock_github_class = MagicMock() @@ -141,7 +148,9 @@ async def test_new_validator_higher_version_available_when_failing_upgrading_sho ) mock_interpreter.return_value = mock_interpreter_class - vc = VersionControl(mock_redis.database) + vc = VersionControl( + mock_redis.database, mock_redis_class.dump_path + ) # Act must_restart = await vc.upgrade() @@ -179,6 +188,7 @@ async def test_current_validator_version_removed_when_upgradring_should_downgrad mock_redis_class.database = AsyncMock(aioredis.Redis) mock_redis_class.get_version = AsyncMock(return_value="2.0.0") mock_redis_class.get_latest_version.return_value = "2.0.0" + mock_redis_class.dump_path = "/etc/redis" mock_redis.return_value = mock_redis_class mock_github_class = MagicMock() @@ -189,7 +199,7 @@ async def test_current_validator_version_removed_when_upgradring_should_downgrad mock_interpreter_class = MagicMock() mock_interpreter.return_value = mock_interpreter_class - vc = VersionControl(mock_redis.database) + vc = VersionControl(mock_redis.database, mock_redis_class.dump_path) # Act must_restart = await vc.upgrade() @@ -226,6 +236,7 @@ async def test_new_higher_redis_version_available_when_upgradring_should_upgrade mock_redis_class = MagicMock() mock_redis_class.get_version = AsyncMock(return_value="2.0.0") mock_redis_class.get_latest_version.return_value = "2.1.0" + mock_redis_class.dump_path = "/etc/redis" mock_redis_class.rollout = AsyncMock() mock_redis.return_value = mock_redis_class @@ -239,7 +250,7 @@ async def test_new_higher_redis_version_available_when_upgradring_should_upgrade mock_create_dump.return_value = AsyncMock() - vc = VersionControl(mock_redis.database) + vc = VersionControl(mock_redis.database, mock_redis_class.dump_path) # Act must_restart = await vc.upgrade() @@ -250,7 +261,7 @@ async def test_new_higher_redis_version_available_when_upgradring_should_upgrade mock_redis_class.rollout.assert_called_once_with("2.0.0", "2.1.0") mock_redis_class.rollback.assert_not_called() mock_create_dump.assert_called_once_with( - "redis-dump-2.0.0", mock_redis_class.database + "/etc/redis/redis-dump-2.0.0", mock_redis_class.database ) mock_restore_dump.assert_not_called() more_create_dump_migrations.assert_called_once() @@ -278,6 +289,7 @@ async def test_new_higher_redis_version_available_when_failing_upgrading_should_ mock_redis_class = MagicMock() mock_redis_class.get_version = AsyncMock(return_value="2.0.0") mock_redis_class.get_latest_version.return_value = "2.1.0" + mock_redis_class.dump_path = "/etc/redis" rollout_side_effect.called = False mock_redis_class.rollout.side_effect = rollout_side_effect mock_redis_class.rollback = AsyncMock() @@ -293,7 +305,7 @@ async def test_new_higher_redis_version_available_when_failing_upgrading_should_ mock_create_dump.return_value = AsyncMock() - vc = VersionControl(mock_redis.database) + vc = VersionControl(mock_redis.database, mock_redis_class.dump_path) # Act must_restart = await vc.upgrade() @@ -304,7 +316,7 @@ async def test_new_higher_redis_version_available_when_failing_upgrading_should_ mock_redis_class.rollout.assert_called_once_with("2.0.0", "2.1.0") mock_redis_class.rollback.assert_called_once_with("2.1.0", "2.0.0") mock_create_dump.assert_called_once_with( - "redis-dump-2.0.0", mock_redis_class.database + "/etc/redis/redis-dump-2.0.0", mock_redis_class.database ) mock_restore_dump.assert_not_called() more_create_dump_migrations.assert_called_once() @@ -332,6 +344,7 @@ async def test_new_higher_redis_version_available_when_failing_upgrading_and_fai mock_redis_class = MagicMock() mock_redis_class.get_version = AsyncMock(return_value="2.0.0") mock_redis_class.get_latest_version.return_value = "2.1.0" + mock_redis_class.dump_path = "/etc/redis" rollout_side_effect.called = False mock_redis_class.rollout.side_effect = rollout_side_effect mock_redis_class.rollback = AsyncMock(return_value=False) @@ -347,7 +360,7 @@ async def test_new_higher_redis_version_available_when_failing_upgrading_and_fai mock_create_dump.return_value = AsyncMock() - vc = VersionControl(mock_redis.database) + vc = VersionControl(mock_redis.database, mock_redis_class.dump_path) # Act must_restart = await vc.upgrade() @@ -358,10 +371,10 @@ async def test_new_higher_redis_version_available_when_failing_upgrading_and_fai mock_redis_class.rollout.assert_called_once_with("2.0.0", "2.1.0") mock_redis_class.rollback.assert_called_once_with("2.1.0", "2.0.0") mock_create_dump.assert_called_once_with( - "redis-dump-2.0.0", mock_redis_class.database + "/etc/redis/redis-dump-2.0.0", mock_redis_class.database ) mock_restore_dump.assert_called_once_with( - "redis-dump-2.0.0", mock_redis_class.database + "/etc/redis/redis-dump-2.0.0", mock_redis_class.database ) more_create_dump_migrations.assert_called_once() more_remove_dump_migrations.assert_has_calls([call(), call()]) @@ -389,6 +402,7 @@ async def test_current_redis_version_removed_when_upgradring_should_downgrade_re mock_redis_class.database = AsyncMock(aioredis.Redis) mock_redis_class.get_version = AsyncMock(return_value="2.1.0") mock_redis_class.get_latest_version.return_value = "2.0.0" + mock_redis_class.dump_path = "/etc/redis" mock_redis_class.rollout = AsyncMock() mock_redis_class.rollback = AsyncMock() mock_redis.return_value = mock_redis_class @@ -403,7 +417,7 @@ async def test_current_redis_version_removed_when_upgradring_should_downgrade_re mock_create_dump.return_value = AsyncMock() - vc = VersionControl(mock_redis.database) + vc = VersionControl(mock_redis.database, mock_redis_class.dump_path) # Act must_restart = await vc.upgrade() @@ -414,7 +428,7 @@ async def test_current_redis_version_removed_when_upgradring_should_downgrade_re mock_redis_class.rollout.assert_not_called() mock_redis_class.rollback.assert_called_once_with("2.1.0", "2.0.0") mock_create_dump.assert_called_once_with( - "redis-dump-2.1.0", mock_redis_class.database + "/etc/redis/redis-dump-2.1.0", mock_redis_class.database ) mock_restore_dump.assert_not_called() more_create_dump_migrations.assert_called_once() @@ -442,6 +456,7 @@ async def test_new_higher_miner_and_redis_version_available_when_upgradring_shou mock_redis_class = MagicMock() mock_redis_class.get_version = AsyncMock(return_value="2.0.0") mock_redis_class.get_latest_version.return_value = "2.1.0" + mock_redis_class.dump_path = "/etc/redis" mock_redis_class.rollout = AsyncMock() mock_redis.return_value = mock_redis_class @@ -455,7 +470,7 @@ async def test_new_higher_miner_and_redis_version_available_when_upgradring_shou mock_create_dump.return_value = AsyncMock() - vc = VersionControl(mock_redis.database) + vc = VersionControl(mock_redis.database, mock_redis_class.dump_path) # Act must_restart = await vc.upgrade() @@ -465,7 +480,7 @@ async def test_new_higher_miner_and_redis_version_available_when_upgradring_shou mock_interpreter_class.upgrade_dependencies.assert_called_once() mock_redis_class.rollout.assert_called_once_with("2.0.0", "2.1.0") mock_create_dump.assert_called_once_with( - "redis-dump-2.0.0", mock_redis_class.database + "/etc/redis/redis-dump-2.0.0", mock_redis_class.database ) mock_restore_dump.assert_not_called() more_create_dump_migrations.assert_called_once() @@ -493,6 +508,7 @@ async def test_new_higher_miner_and_redis_version_available_when_validator_uprad mock_redis_class = MagicMock() mock_redis_class.get_version = AsyncMock(return_value="2.0.0") mock_redis_class.get_latest_version.return_value = "2.1.0" + mock_redis_class.dump_path = "/etc/redis" mock_redis_class.rollout = AsyncMock() rollout_side_effect.called = False mock_redis_class.rollout.side_effect = rollout_side_effect @@ -509,7 +525,7 @@ async def test_new_higher_miner_and_redis_version_available_when_validator_uprad mock_create_dump.return_value = AsyncMock() - vc = VersionControl(mock_redis.database) + vc = VersionControl(mock_redis.database, mock_redis_class.dump_path) # Act must_restart = await vc.upgrade() @@ -520,7 +536,7 @@ async def test_new_higher_miner_and_redis_version_available_when_validator_uprad mock_redis_class.rollout.assert_called_once_with("2.0.0", "2.1.0") mock_redis_class.rollback.assert_called_once_with("2.1.0", "2.0.0") mock_create_dump.assert_called_once_with( - "redis-dump-2.0.0", mock_redis_class.database + "/etc/redis/redis-dump-2.0.0", mock_redis_class.database ) mock_restore_dump.assert_not_called() more_create_dump_migrations.assert_called_once() diff --git a/tests/unit_tests/subnet/version/test_get_latest_version.py b/tests/unit_tests/subnet/version/test_get_latest_version.py new file mode 100644 index 00000000..e04e5697 --- /dev/null +++ b/tests/unit_tests/subnet/version/test_get_latest_version.py @@ -0,0 +1,99 @@ +import unittest +from unittest.mock import patch + +from subnet.version.github_controller import Github + + +class TestGithubController(unittest.IsolatedAsyncioTestCase): + @patch("requests.get") + @patch("codecs.open") + def test_request_latest_successful_should_return_the_latest_version( + self, mock_open, mock_request + ): + # Arrange + mock_open.return_value.__enter__.return_value.read.return_value = "" + + mock_request.return_value.status_code = 200 + mock_request.return_value.json.return_value = {"tag_name": "v2.2.3"} + + github = Github() + + # Act + result = github.get_latest_version() + + # Assert + assert "2.2.3" == result + + @patch("requests.get") + @patch("codecs.open") + def test_request_latest_failed_and_no_cached_version_exist_should_return_none( + self, mock_open, mock_request + ): + # Arrange + mock_open.return_value.__enter__.return_value.read.return_value = "" + + mock_request.return_value.status_code = 300 + mock_request.return_value.json.return_value = {"tag_name": "v2.2.3"} + + github = Github() + + # Act + result = github.get_latest_version() + + # Assert + assert None == result + + @patch("requests.get") + @patch("codecs.open") + def test_request_latest_failed_and_a_cached_version_exist_should_return_the_cached_version( + self, mock_open, mock_request + ): + # Arrange + mock_open.return_value.__enter__.return_value.read.return_value = "" + + mock_request.return_value.status_code = 200 + mock_request.return_value.json.return_value = {"tag_name": "v2.2.3"} + + github = Github() + + # Act + result1 = github.get_latest_version() + + # Assert + assert "2.2.3" == result1 + + # Arrange + mock_request.return_value.status_code = 300 + mock_request.return_value.json.return_value = {"tag_name": "v2.2.4"} + + # Act + result2 = github.get_latest_version() + + assert "2.2.3" == result2 + + @patch("requests.get") + @patch("codecs.open") + def test_request_latest_throw_exception_and_a_cached_version_exist_should_return_the_cached_version( + self, mock_open, mock_request + ): + # Arrange + mock_open.return_value.__enter__.return_value.read.return_value = "" + + mock_request.return_value.status_code = 200 + mock_request.return_value.json.return_value = {"tag_name": "v2.2.3"} + + github = Github() + + # Act + result1 = github.get_latest_version() + + # Assert + assert "2.2.3" == result1 + + # Arrange + mock_request.return_value.json.return_value = ValueError("Simulated error") + + # Act + result2 = github.get_latest_version() + + assert "2.2.3" == result2 diff --git a/tests/unit_tests/subnet/version/test_get_migrations.py b/tests/unit_tests/subnet/version/test_get_migrations.py index c0471602..f390e707 100644 --- a/tests/unit_tests/subnet/version/test_get_migrations.py +++ b/tests/unit_tests/subnet/version/test_get_migrations.py @@ -6,7 +6,7 @@ class TestUtilVersionControl(unittest.IsolatedAsyncioTestCase): @patch("os.listdir") - async def test_no_migration_available_should_return_an_empty_list( + async def test_forward_order_with_order_with_no_migration_available_should_return_an_empty_list( self, mock_listdir ): # Arrange @@ -19,7 +19,7 @@ async def test_no_migration_available_should_return_an_empty_list( assert 0 == len(result) @patch("os.listdir") - async def test_migration_available_should_return_a_list_in_the_right_order( + async def test_forward_order_with_migration_available_should_return_a_list_in_the_right_order( self, mock_listdir ): # Arrange @@ -32,6 +32,75 @@ async def test_migration_available_should_return_a_list_in_the_right_order( # Act result = get_migrations() + # Assert + assert 3 == len(result) + assert (200, "2.0.0", "migration-2.0.0.py") == result[0] + assert (210, "2.1.0", "migration-2.1.0.py") == result[1] + assert (211, "2.1.1", "migration-2.1.1.py") == result[2] + + @patch("os.listdir") + async def test_forward_order_with_migration_available_and_filter_applied_should_return_a_list_in_the_right_order( + self, mock_listdir + ): + # Arrange + mock_listdir.return_value = [ + "migration-2.1.0.py", + "migration-2.0.0.py", + "migration-2.1.1.py", + ] + + # Act + result = get_migrations(filter_lambda=lambda x: x[0] > 200 and x[0] <= 211) + + # Assert + assert 2 == len(result) + assert (210, "2.1.0", "migration-2.1.0.py") == result[0] + assert (211, "2.1.1", "migration-2.1.1.py") == result[1] + + @patch("os.listdir") + async def test_forward_order_with_migration_available_when_few_does_match_the_expected_pattern_should_return_a_list_without_these_wrong_formatted_files( + self, mock_listdir + ): + # Arrange + mock_listdir.return_value = [ + "migration2.1.1.py", + "migrations-2.0.0.py", + "migration-210.py", + ] + + # Act + result = get_migrations() + + # Assert + assert 0 == len(result) + + @patch("os.listdir") + async def test_reverse_order_with_no_migration_available_should_return_an_empty_list( + self, mock_listdir + ): + # Arrange + mock_listdir.return_value = [] + + # Act + result = get_migrations(reverse=True) + + # Assert + assert 0 == len(result) + + @patch("os.listdir") + async def test_reverse_order_with_migration_available_should_return_a_list_in_the_right_order( + self, mock_listdir + ): + # Arrange + mock_listdir.return_value = [ + "migration-2.1.0.py", + "migration-2.0.0.py", + "migration-2.1.1.py", + ] + + # Act + result = get_migrations(reverse=True) + # Assert assert 3 == len(result) assert (211, "2.1.1", "migration-2.1.1.py") == result[0] @@ -39,7 +108,28 @@ async def test_migration_available_should_return_a_list_in_the_right_order( assert (200, "2.0.0", "migration-2.0.0.py") == result[2] @patch("os.listdir") - async def test_migration_available_when_few_does_match_the_expected_pattern_should_return_a_list_without_these_wrong_formatted_files( + async def test_reverse_order_with_migration_available_and_filter_applied_should_return_a_list_in_the_right_order( + self, mock_listdir + ): + # Arrange + mock_listdir.return_value = [ + "migration-2.1.0.py", + "migration-2.0.0.py", + "migration-2.1.1.py", + ] + + # Act + result = get_migrations( + reverse=True, filter_lambda=lambda x: x[0] > 200 and x[0] <= 211 + ) + + # Assert + assert 2 == len(result) + assert (211, "2.1.1", "migration-2.1.1.py") == result[0] + assert (210, "2.1.0", "migration-2.1.0.py") == result[1] + + @patch("os.listdir") + async def test_reverse_order_with_migration_available_when_few_does_match_the_expected_pattern_should_return_a_list_without_these_wrong_formatted_files( self, mock_listdir ): # Arrange @@ -50,7 +140,7 @@ async def test_migration_available_when_few_does_match_the_expected_pattern_shou ] # Act - result = get_migrations() + result = get_migrations(reverse=True) # Assert assert 0 == len(result)