diff --git a/build.sh b/build.sh index e7ff502..1d4b67f 100755 --- a/build.sh +++ b/build.sh @@ -5,10 +5,10 @@ clear # Define Resolver > Has to be one of 'fileResolver'/'pythonResolver'/'cachedResolver'/'httpResolver' # export RESOLVER_NAME=cachedResolver # Clear existing build data and invoke cmake -rm -R build -rm -R dist +#rm -R build +#rm -R dist set -e # Exit on error cmake . -B build -cmake --build build --clean-first # make clean all +cmake --build build # --clean-first # make clean all cmake --install build # make install ctest -VV --test-dir build \ No newline at end of file diff --git a/src/CachedResolver/CMakeLists.txt b/src/CachedResolver/CMakeLists.txt index d488306..fbf4384 100644 --- a/src/CachedResolver/CMakeLists.txt +++ b/src/CachedResolver/CMakeLists.txt @@ -68,8 +68,9 @@ add_library(${AR_CACHEDRESOLVER_TARGET_PYTHON} SHARED module.cpp moduleDeps.cpp - resolverTokens.cpp - resolverContext.cpp + #resolver.cpp + #resolverTokens.cpp + #resolverContext.cpp wrapResolver.cpp wrapResolverContext.cpp wrapResolverTokens.cpp diff --git a/src/CachedResolver/PythonExpose.py b/src/CachedResolver/PythonExpose.py index 3bc6420..ef7dc44 100644 --- a/src/CachedResolver/PythonExpose.py +++ b/src/CachedResolver/PythonExpose.py @@ -4,8 +4,7 @@ from functools import wraps from pxr import Ar -# This import is needed so that our methods below know what a CachedResolver.Context is! -from usdAssetResolver import CachedResolver + # Init logger logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y/%m/%d %I:%M:%S%p") @@ -39,6 +38,28 @@ def reset(cls, current_directory_path=""): cls.current_directory_path = current_directory_path +class Resolver: + + @staticmethod + @log_function_args + def CreateRelativePathIdentifier(resolver, anchoredAssetPath, assetPath, anchorAssetPath, ): + """Returns an identifier for the asset specified by assetPath. + If anchorAssetPath is not empty, it is the resolved asset path + that assetPath should be anchored to if it is a relative path. + Args: + resolver (CachedResolver): The resolver + anchoredAssetPath (str): The anchored asset path, this has to be used as the cached key. + assetPath (str): An unresolved asset path. + anchorAssetPath (Ar.ResolvedPath): A resolved anchor path. + + Returns: + str: The identifier. + """ + remappedRelativePathIdentifier = f"{assetPath[2:]}?{anchorAssetPath}" + resolver.AddCachedRelativePathIdentifierPair(anchoredAssetPath, remappedRelativePathIdentifier) + return remappedRelativePathIdentifier + + class ResolverContext: @staticmethod @@ -62,12 +83,12 @@ def Initialize(context): @staticmethod @log_function_args - def ResolveAndCache(assetPath, context): + def ResolveAndCache(context, assetPath): """Return the resolved path for the given assetPath or an empty ArResolvedPath if no asset exists at that path. Args: - assetPath (str): An unresolved asset path. context (CachedResolverContext): The active context. + assetPath (str): An unresolved asset path. Returns: str: The resolved path string. If it points to a non-existent file, it will be resolved to an empty ArResolvedPath internally, but will diff --git a/src/CachedResolver/resolver.cpp b/src/CachedResolver/resolver.cpp index 2a53751..01b8071 100644 --- a/src/CachedResolver/resolver.cpp +++ b/src/CachedResolver/resolver.cpp @@ -7,6 +7,7 @@ #include "pxr/base/arch/systemInfo.h" #include "pxr/base/tf/fileUtils.h" #include "pxr/base/tf/pathUtils.h" +#include "pxr/base/tf/pyInvoke.h" #include "pxr/base/tf/staticTokens.h" #include "pxr/base/tf/stringUtils.h" #include "pxr/usd/sdf/layer.h" @@ -18,10 +19,18 @@ #include #include #include - +#include +#include #include #include +/* +Safety-wise we lock via a mutex when we re-rout the relative path lookup to a Python call. +In theory we probably don't need this, as our Python call does this anyway. +See the _CreateIdentifier method for more information. +*/ +static std::mutex g_resolver_create_identifier_mutex; + namespace python = AR_BOOST_NAMESPACE::python; PXR_NAMESPACE_OPEN_SCOPE @@ -80,6 +89,38 @@ CachedResolver::CachedResolver() = default; CachedResolver::~CachedResolver() = default; +void CachedResolver::AddCachedRelativePathIdentifierPair(const std::string& sourceStr, const std::string& targetStr){ + auto cache_find = cachedRelativePathIdentifierPairs.find(sourceStr); + if(cache_find != cachedRelativePathIdentifierPairs.end()){ + cache_find->second = targetStr; + }else{ + cachedRelativePathIdentifierPairs.insert(std::pair(sourceStr,targetStr)); + } +} + + +void CachedResolver::RemoveCachedRelativePathIdentifierByKey(const std::string& sourceStr){ + const auto &it = cachedRelativePathIdentifierPairs.find(sourceStr); + if (it != cachedRelativePathIdentifierPairs.end()){ + cachedRelativePathIdentifierPairs.erase(it); + } +} + + +void CachedResolver::RemoveCachedRelativePathIdentifierByValue(const std::string& targetStr){ + for (auto it = cachedRelativePathIdentifierPairs.cbegin(); it != cachedRelativePathIdentifierPairs.cend();) + { + if (it->second == targetStr) + { + cachedRelativePathIdentifierPairs.erase(it++); + } + else + { + ++it; + } + } +} + std::string CachedResolver::_CreateIdentifier( const std::string& assetPath, @@ -97,11 +138,46 @@ CachedResolver::_CreateIdentifier( } const std::string anchoredAssetPath = _AnchorRelativePath(anchorAssetPath, assetPath); - + // Anchor non file path based identifiers and see if a file exists. + // This is mostly for debugging as it allows us to add a file relative to our + // anchor directory that has a higher priority than our (usually unanchored) + // resolved asset path. if (_IsNotFilePath(assetPath) && Resolve(anchoredAssetPath).empty()) { return TfNormPath(assetPath); } + // Re-direct to Python to allow optional re-routing of relative paths + // through the resolver. + if (true) { + if (_IsFileRelativePath(assetPath)) { + auto cache_find = this->cachedRelativePathIdentifierPairs.find(anchorAssetPath); + if(cache_find != this->cachedRelativePathIdentifierPairs.end()){ + return cache_find->second; + }else{ + std::string pythonResult; + { + /* + We optionally re-route relative file paths to be pre-formatted in Python. + This allows us to optionally override relative paths too and not only + non file path identifiers. + Please see the CachedResolverContext::ResolveAndCachePair method for + risks (as this does the same hacky workaround) + and the Python Resolver.CreateRelativePathIdentified method on how to use this. + */ + const std::lock_guard lock(g_resolver_create_identifier_mutex); + int state = TfPyInvokeAndExtract(DEFINE_STRING(AR_CACHEDRESOLVER_USD_PYTHON_EXPOSE_MODULE_NAME), + "Resolver.CreateRelativePathIdentifier", + &pythonResult, AR_BOOST_NAMESPACE::ref(*this), anchoredAssetPath, assetPath, anchorAssetPath); + if (!state) { + std::cerr << "Failed to call Resolver.CreateRelativePathIdentifier in " << DEFINE_STRING(AR_CACHEDRESOLVER_USD_PYTHON_EXPOSE_MODULE_NAME) << ".py. "; + std::cerr << "Please verify that the python code is valid!" << std::endl; + pythonResult = TfNormPath(anchoredAssetPath); + } + } + return pythonResult; + } + } + } return TfNormPath(anchoredAssetPath); } @@ -137,51 +213,51 @@ CachedResolver::_Resolve( if (SdfLayer::IsAnonymousLayerIdentifier(assetPath)){ return ArResolvedPath(assetPath); } - if (_IsRelativePath(assetPath)) { - if (this->_IsContextDependentPath(assetPath)) { - const CachedResolverContext* contexts[2] = {this->_GetCurrentContextPtr(), &_fallbackContext}; - for (const CachedResolverContext* ctx : contexts) { - if (ctx) { - // Search for mapping pairs - auto &mappingPairs = ctx->GetMappingPairs(); - auto map_find = mappingPairs.find(assetPath); - if(map_find != mappingPairs.end()){ - ArResolvedPath resolvedPath = _ResolveAnchored(this->emptyString, map_find->second); - return resolvedPath; - // Assume that a map hit is always valid. - // if (resolvedPath) { - // return resolvedPath; - // } - } - // Search for cached pairs - auto &cachedPairs = ctx->GetCachingPairs(); - auto cache_find = cachedPairs.find(assetPath); - if(cache_find != cachedPairs.end()){ - ArResolvedPath resolvedPath = _ResolveAnchored(this->emptyString, cache_find->second); - return resolvedPath; - // Assume that a cache hit is always valid. - // if (resolvedPath) { - // return resolvedPath; - // } - } - // Perform query if caches don't have a hit. - TF_DEBUG(CACHEDRESOLVER_RESOLVER).Msg("Resolver::_Resolve('%s') -> No cache hit, switching to Python query\n", assetPath.c_str()); - /* - We perform the resource/thread lock in the context itself to - allow for resolver multithreading with different contexts. - See .ResolveAndCachePair for more information. - */ - ArResolvedPath resolvedPath = _ResolveAnchored(this->emptyString, ctx->ResolveAndCachePair(assetPath)); - if (resolvedPath) { - return resolvedPath; - } - // Only try the first valid context. - break; + + if (this->_IsContextDependentPath(assetPath)) { + const CachedResolverContext* contexts[2] = {this->_GetCurrentContextPtr(), &_fallbackContext}; + for (const CachedResolverContext* ctx : contexts) { + if (ctx) { + // Search for mapping pairs + auto &mappingPairs = ctx->GetMappingPairs(); + auto map_find = mappingPairs.find(assetPath); + if(map_find != mappingPairs.end()){ + ArResolvedPath resolvedPath = _ResolveAnchored(this->emptyString, map_find->second); + return resolvedPath; + // Assume that a map hit is always valid. + // if (resolvedPath) { + // return resolvedPath; + // } + } + // Search for cached pairs + auto &cachedPairs = ctx->GetCachingPairs(); + auto cache_find = cachedPairs.find(assetPath); + if(cache_find != cachedPairs.end()){ + ArResolvedPath resolvedPath = _ResolveAnchored(this->emptyString, cache_find->second); + return resolvedPath; + // Assume that a cache hit is always valid. + // if (resolvedPath) { + // return resolvedPath; + // } } + // Perform query if caches don't have a hit. + TF_DEBUG(CACHEDRESOLVER_RESOLVER).Msg("Resolver::_Resolve('%s') -> No cache hit, switching to Python query\n", assetPath.c_str()); + /* + We perform the resource/thread lock in the context itself to + allow for resolver multithreading with different contexts. + See .ResolveAndCachePair for more information. + */ + ArResolvedPath resolvedPath = _ResolveAnchored(this->emptyString, ctx->ResolveAndCachePair(assetPath)); + if (resolvedPath) { + return resolvedPath; + } + // Only try the first valid context. + break; } } return ArResolvedPath(); } + return _ResolveAnchored(std::string(), assetPath); } diff --git a/src/CachedResolver/resolver.h b/src/CachedResolver/resolver.h index e24406c..aa2b5d0 100644 --- a/src/CachedResolver/resolver.h +++ b/src/CachedResolver/resolver.h @@ -27,53 +27,52 @@ class CachedResolver final : public ArResolver public: AR_CACHEDRESOLVER_API CachedResolver(); - AR_CACHEDRESOLVER_API virtual ~CachedResolver(); + AR_CACHEDRESOLVER_API + void AddCachedRelativePathIdentifierPair(const std::string& sourceStr, const std::string& targetStr); + AR_CACHEDRESOLVER_API + void RemoveCachedRelativePathIdentifierByKey(const std::string& sourceStr); + AR_CACHEDRESOLVER_API + void RemoveCachedRelativePathIdentifierByValue(const std::string& targetStr); + AR_CACHEDRESOLVER_API + const std::map& GetCachedRelativePathIdentifierPairs() const { return cachedRelativePathIdentifierPairs; } + AR_CACHEDRESOLVER_API + void ClearCachedRelativePathIdentifierPairs() { cachedRelativePathIdentifierPairs.clear(); } protected: AR_CACHEDRESOLVER_API std::string _CreateIdentifier( const std::string& assetPath, const ArResolvedPath& anchorAssetPath) const final; - AR_CACHEDRESOLVER_API std::string _CreateIdentifierForNewAsset( const std::string& assetPath, const ArResolvedPath& anchorAssetPath) const final; - AR_CACHEDRESOLVER_API ArResolvedPath _Resolve( const std::string& assetPath) const final; - AR_CACHEDRESOLVER_API ArResolvedPath _ResolveForNewAsset( const std::string& assetPath) const final; - AR_CACHEDRESOLVER_API ArResolverContext _CreateDefaultContext() const final; - AR_CACHEDRESOLVER_API ArResolverContext _CreateDefaultContextForAsset( const std::string& assetPath) const final; - AR_CACHEDRESOLVER_API bool _IsContextDependentPath( const std::string& assetPath) const final; - AR_CACHEDRESOLVER_API void _RefreshContext( const ArResolverContext& context) final; - AR_CACHEDRESOLVER_API ArTimestamp _GetModificationTimestamp( const std::string& assetPath, const ArResolvedPath& resolvedPath) const final; - AR_CACHEDRESOLVER_API std::shared_ptr _OpenAsset( const ArResolvedPath& resolvedPath) const final; - AR_CACHEDRESOLVER_API std::shared_ptr _OpenAssetForWrite( const ArResolvedPath& resolvedPath, @@ -83,6 +82,7 @@ class CachedResolver final : public ArResolver const CachedResolverContext* _GetCurrentContextPtr() const; CachedResolverContext _fallbackContext; const std::string emptyString{}; + std::map cachedRelativePathIdentifierPairs; }; PXR_NAMESPACE_CLOSE_SCOPE diff --git a/src/CachedResolver/resolverContext.cpp b/src/CachedResolver/resolverContext.cpp index 1f9d629..4d34212 100644 --- a/src/CachedResolver/resolverContext.cpp +++ b/src/CachedResolver/resolverContext.cpp @@ -177,27 +177,27 @@ void CachedResolverContext::RemoveMappingByValue(const std::string& targetStr){ } void CachedResolverContext::AddCachingPair(const std::string& sourceStr, const std::string& targetStr){ - auto cache_find = data->cachedPairs.find(sourceStr); - if(cache_find != data->cachedPairs.end()){ + auto cache_find = data->cachingPairs.find(sourceStr); + if(cache_find != data->cachingPairs.end()){ cache_find->second = targetStr; }else{ - data->cachedPairs.insert(std::pair(sourceStr,targetStr)); + data->cachingPairs.insert(std::pair(sourceStr,targetStr)); } } void CachedResolverContext::RemoveCachingByKey(const std::string& sourceStr){ - const auto &it = data->cachedPairs.find(sourceStr); - if (it != data->cachedPairs.end()){ - data->cachedPairs.erase(it); + const auto &it = data->cachingPairs.find(sourceStr); + if (it != data->cachingPairs.end()){ + data->cachingPairs.erase(it); } } void CachedResolverContext::RemoveCachingByValue(const std::string& targetStr){ - for (auto it = data->cachedPairs.cbegin(); it != data->cachedPairs.cend();) + for (auto it = data->cachingPairs.cbegin(); it != data->cachingPairs.cend();) { if (it->second == targetStr) { - data->cachedPairs.erase(it++); + data->cachingPairs.erase(it++); } else { @@ -224,7 +224,7 @@ const std::string CachedResolverContext::ResolveAndCachePair(const std::string& int state = TfPyInvokeAndExtract(DEFINE_STRING(AR_CACHEDRESOLVER_USD_PYTHON_EXPOSE_MODULE_NAME), "ResolverContext.ResolveAndCache", - &pythonResult, assetPath, this); + &pythonResult, this, assetPath); if (!state) { std::cerr << "Failed to call Resolver.ResolveAndCache in " << DEFINE_STRING(AR_CACHEDRESOLVER_USD_PYTHON_EXPOSE_MODULE_NAME) << ".py. "; std::cerr << "Please verify that the python code is valid!" << std::endl; diff --git a/src/CachedResolver/resolverContext.h b/src/CachedResolver/resolverContext.h index 22eca12..ec6fdb0 100644 --- a/src/CachedResolver/resolverContext.h +++ b/src/CachedResolver/resolverContext.h @@ -25,7 +25,7 @@ struct CachedResolverContextInternalData { std::string mappingFilePath; std::map mappingPairs; - std::map cachedPairs; + std::map cachingPairs; }; class CachedResolverContext @@ -77,17 +77,15 @@ class CachedResolverContext AR_CACHEDRESOLVER_API void RemoveCachingByValue(const std::string& targetStr); AR_CACHEDRESOLVER_API - const std::map& GetCachingPairs() const { return data->cachedPairs; } + const std::map& GetCachingPairs() const { return data->cachingPairs; } AR_CACHEDRESOLVER_API - void ClearCachingPairs() { data->cachedPairs.clear(); } + void ClearCachingPairs() { data->cachingPairs.clear(); } AR_CACHEDRESOLVER_API const std::string ResolveAndCachePair(const std::string& assetPath) const; private: - // Vars std::shared_ptr data = std::make_shared(); std::string emptyString{""}; - // Methods bool _GetMappingPairsFromUsdFile(const std::string& filePath); }; diff --git a/src/CachedResolver/testenv/testCachedResolver.py b/src/CachedResolver/testenv/testCachedResolver.py index 4f2d3f7..f47d6af 100644 --- a/src/CachedResolver/testenv/testCachedResolver.py +++ b/src/CachedResolver/testenv/testCachedResolver.py @@ -282,7 +282,6 @@ def test_ResolverCachingMechanism(self): self.assertEqual(ctx.GetMappingPairs(), {asset_a_identifier: asset_c_layer_file_path}) self.assertEqual(PythonExpose.UnitTestHelper.context_initialize_call_counter, 2) - def test_ResolveWithContext(self): with tempfile.TemporaryDirectory() as temp_dir_path: # Create files diff --git a/src/CachedResolver/wrapResolver.cpp b/src/CachedResolver/wrapResolver.cpp index 3388bee..6f9f2e3 100644 --- a/src/CachedResolver/wrapResolver.cpp +++ b/src/CachedResolver/wrapResolver.cpp @@ -1,6 +1,7 @@ #include "resolver.h" #include +#include #include "boost_include_wrapper.h" #include BOOST_INCLUDE(python/class.hpp) @@ -17,5 +18,10 @@ wrapResolver() class_, AR_BOOST_NAMESPACE::noncopyable> ("Resolver", no_init) + .def("GetCachedRelativePathIdentifierPairs", &This::AddCachedRelativePathIdentifierPair, return_value_policy(), "Returns all cached relative path identifier pairs as a dict") + .def("AddCachedRelativePathIdentifierPair", &This::AddCachedRelativePathIdentifierPair, "Remove a cached relative path identifier pair by value") + .def("RemoveCachedRelativePathIdentifierByKey", &This::RemoveCachedRelativePathIdentifierByKey, "Add a cached relative path identifier pair") + .def("RemoveCachedRelativePathIdentifierByValue", &This::RemoveCachedRelativePathIdentifierByValue, "Remove a cached relative path identifier pair by key") + .def("ClearCachedRelativePathIdentifierPairs", &This::ClearCachedRelativePathIdentifierPairs, "Clear all cached relative path identifier pairs") ; } \ No newline at end of file diff --git a/src/CachedResolver/wrapResolverContext.cpp b/src/CachedResolver/wrapResolverContext.cpp index dbd53ac..9238987 100644 --- a/src/CachedResolver/wrapResolverContext.cpp +++ b/src/CachedResolver/wrapResolverContext.cpp @@ -36,7 +36,7 @@ wrapResolverContext() { using This = CachedResolverContext; - class_("ResolverContext", no_init) + class_("ResolverContext", no_init) .def(init<>()) .def(init(args("mappingFile"))) .def(self == self)