Skip to content

Commit

Permalink
Model cache...
Browse files Browse the repository at this point in the history
  • Loading branch information
jmchilton committed Jul 7, 2024
1 parent e6e6410 commit 9e09a86
Show file tree
Hide file tree
Showing 7 changed files with 66 additions and 13 deletions.
6 changes: 3 additions & 3 deletions lib/galaxy/config/schemas/tool_shed_config_schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -102,12 +102,12 @@ mapping:
the repositories and tools within the Tool Shed given that you specify
the following two config options.
tool_state_cache_dir:
model_cache_dir:
type: str
default: database/tool_state_cache
default: database/model_cache
required: false
desc: |
Cache directory for tool state.
Cache directory for Pydantic model objects.
repo_name_boost:
type: float
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,28 @@
from typing import (
Any,
Dict,
Generic,
Optional,
Type,
TypeVar,
)

from pydantic import BaseModel

from galaxy.util.hash_util import md5_hash_str


RAW_CACHED_JSON = Dict[str, Any]


class ToolStateCache:
def hash_model(model_class: Type[BaseModel]) -> str:
return md5_hash_str(json.dumps(model_class.model_json_schema()))


M = TypeVar("M", bound=BaseModel)


class ModelCache(Generic[M]):
_cache_directory: str

def __init__(self, cache_directory: str):
Expand Down
8 changes: 5 additions & 3 deletions lib/tool_shed/managers/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
Tuple,
)

from pydantic import BaseModel

from galaxy import exceptions
from galaxy.exceptions import (
InternalServerError,
Expand Down Expand Up @@ -148,12 +150,12 @@ def get_repository_metadata_tool_dict(
def tool_input_models_cached_for(
trans: ProvidesRepositoriesContext, trs_tool_id: str, tool_version: str, repository_clone_url: Optional[str] = None
) -> ToolParameterBundleModel:
tool_state_cache = trans.app.tool_state_cache
raw_json = tool_state_cache.get_cache_entry_for(trs_tool_id, tool_version)
model_cache = trans.app.model_cache
raw_json = model_cache.get_cache_entry_for(trs_tool_id, tool_version)
if raw_json is not None:
return tool_parameter_bundle_from_json(raw_json)
bundle = tool_input_models_for(trans, trs_tool_id, tool_version, repository_clone_url=repository_clone_url)
tool_state_cache.insert_cache_entry_for(trs_tool_id, tool_version, bundle.dict())
model_cache.insert_cache_entry_for(trs_tool_id, tool_version, bundle.dict())
return bundle


Expand Down
4 changes: 2 additions & 2 deletions lib/tool_shed/structured_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from galaxy.structured_app import BasicSharedApp

if TYPE_CHECKING:
from tool_shed.managers.tool_state_cache import ToolStateCache
from tool_shed.managers.model_cache import ModelCache
from tool_shed.repository_registry import Registry as RepositoryRegistry
from tool_shed.repository_types.registry import Registry as RepositoryTypesRegistry
from tool_shed.util.hgweb_config import HgWebConfigManager
Expand All @@ -17,4 +17,4 @@ class ToolShedApp(BasicSharedApp):
repository_registry: "RepositoryRegistry"
hgweb_config_manager: "HgWebConfigManager"
security_agent: "CommunityRBACAgent"
tool_state_cache: "ToolStateCache"
model_cache: "ModelCache"
4 changes: 2 additions & 2 deletions lib/tool_shed/webapp/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
from galaxy.structured_app import BasicSharedApp
from galaxy.web_stack import application_stack_instance
from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager
from tool_shed.managers.tool_state_cache import ToolStateCache
from tool_shed.managers.model_cache import ModelCache
from tool_shed.structured_app import ToolShedApp
from tool_shed.util.hgweb_config import hgweb_config_manager
from tool_shed.webapp.model.migrations import verify_database
Expand Down Expand Up @@ -84,7 +84,7 @@ def __init__(self, **kwd) -> None:
self._register_singleton(SharedModelMapping, model)
self._register_singleton(mapping.ToolShedModelMapping, model)
self._register_singleton(scoped_session, self.model.context)
self.tool_state_cache = ToolStateCache(self.config.tool_state_cache_dir)
self.model_cache = ModelCache(self.config.model_cache_dir)
self.user_manager = self._register_singleton(UserManager, UserManager(self, app_type="tool_shed"))
self.api_keys_manager = self._register_singleton(ApiKeyManager)
# initialize the Tool Shed tag handler.
Expand Down
4 changes: 2 additions & 2 deletions test/unit/tool_shed/_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from galaxy.util import safe_makedirs
from tool_shed.context import ProvidesRepositoriesContext
from tool_shed.managers.repositories import upload_tar_and_set_metadata
from tool_shed.managers.tool_state_cache import ToolStateCache
from tool_shed.managers.model_cache import ModelCache
from tool_shed.managers.users import create_user
from tool_shed.repository_types import util as rt_util
from tool_shed.repository_types.registry import Registry as RepositoryTypesRegistry
Expand Down Expand Up @@ -81,7 +81,7 @@ def __init__(self, temp_directory=None):
self.config = TestToolShedConfig(temp_directory)
self.security = IdEncodingHelper(id_secret=self.config.id_secret)
self.repository_registry = tool_shed.repository_registry.Registry(self)
self.tool_state_cache = ToolStateCache(os.path.join(temp_directory, "tool_state_cache"))
self.model_cache = ModelCache(os.path.join(temp_directory, "model_cache"))

@property
def security_agent(self):
Expand Down
36 changes: 36 additions & 0 deletions test/unit/tool_shed/test_model_cache.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
from pydantic import BaseModel, ConfigDict

from tool_shed.managers.model_cache import hash_model


class Moo(BaseModel):
foo: int


class MooLike(BaseModel):
model_config = ConfigDict(title="Moo")
foo: int


class NewMoo(BaseModel):
model_config = ConfigDict(title="Moo")
foo: int
new_prop: str


def test_hash():
hash_moo_1 = hash_model(Moo)
hash_moo_2 = hash_model(Moo)
assert hash_moo_1 == hash_moo_2


def test_hash_by_value():
hash_moo_1 = hash_model(Moo)
hash_moo_like = hash_model(MooLike)
assert hash_moo_1 == hash_moo_like


def test_hash_different_on_updates():
hash_moo_1 = hash_model(Moo)
hash_moo_new = hash_model(NewMoo)
assert hash_moo_1 != hash_moo_new

0 comments on commit 9e09a86

Please sign in to comment.