Skip to content

Commit

Permalink
Init adding relative identifier exposure
Browse files Browse the repository at this point in the history
  • Loading branch information
@LucaScheller
LucaScheller committed Nov 21, 2023
1 parent bdb93ba commit 5c53805
Show file tree
Hide file tree
Showing 10 changed files with 179 additions and 78 deletions.
6 changes: 3 additions & 3 deletions build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ clear
# Define Resolver > Has to be one of 'fileResolver'/'pythonResolver'/'cachedResolver'/'httpResolver'
# export RESOLVER_NAME=cachedResolver
# Clear existing build data and invoke cmake
rm -R build
rm -R dist
#rm -R build
#rm -R dist
set -e # Exit on error
cmake . -B build
cmake --build build --clean-first # make clean all
cmake --build build # --clean-first # make clean all
cmake --install build # make install
ctest -VV --test-dir build
5 changes: 3 additions & 2 deletions src/CachedResolver/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,9 @@ add_library(${AR_CACHEDRESOLVER_TARGET_PYTHON}
SHARED
module.cpp
moduleDeps.cpp
resolverTokens.cpp
resolverContext.cpp
#resolver.cpp
#resolverTokens.cpp
#resolverContext.cpp
wrapResolver.cpp
wrapResolverContext.cpp
wrapResolverTokens.cpp
Expand Down
29 changes: 25 additions & 4 deletions src/CachedResolver/PythonExpose.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@
from functools import wraps

from pxr import Ar
# This import is needed so that our methods below know what a CachedResolver.Context is!
from usdAssetResolver import CachedResolver


# Init logger
logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y/%m/%d %I:%M:%S%p")
Expand Down Expand Up @@ -39,6 +38,28 @@ def reset(cls, current_directory_path=""):
cls.current_directory_path = current_directory_path


class Resolver:

@staticmethod
@log_function_args
def CreateRelativePathIdentifier(resolver, anchoredAssetPath, assetPath, anchorAssetPath, ):
"""Returns an identifier for the asset specified by assetPath.
If anchorAssetPath is not empty, it is the resolved asset path
that assetPath should be anchored to if it is a relative path.
Args:
resolver (CachedResolver): The resolver
anchoredAssetPath (str): The anchored asset path, this has to be used as the cached key.
assetPath (str): An unresolved asset path.
anchorAssetPath (Ar.ResolvedPath): A resolved anchor path.
Returns:
str: The identifier.
"""
remappedRelativePathIdentifier = f"{assetPath[2:]}?{anchorAssetPath}"
resolver.AddCachedRelativePathIdentifierPair(anchoredAssetPath, remappedRelativePathIdentifier)
return remappedRelativePathIdentifier


class ResolverContext:

@staticmethod
Expand All @@ -62,12 +83,12 @@ def Initialize(context):

@staticmethod
@log_function_args
def ResolveAndCache(assetPath, context):
def ResolveAndCache(context, assetPath):
"""Return the resolved path for the given assetPath or an empty
ArResolvedPath if no asset exists at that path.
Args:
assetPath (str): An unresolved asset path.
context (CachedResolverContext): The active context.
assetPath (str): An unresolved asset path.
Returns:
str: The resolved path string. If it points to a non-existent file,
it will be resolved to an empty ArResolvedPath internally, but will
Expand Down
160 changes: 118 additions & 42 deletions src/CachedResolver/resolver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include "pxr/base/arch/systemInfo.h"
#include "pxr/base/tf/fileUtils.h"
#include "pxr/base/tf/pathUtils.h"
#include "pxr/base/tf/pyInvoke.h"
#include "pxr/base/tf/staticTokens.h"
#include "pxr/base/tf/stringUtils.h"
#include "pxr/usd/sdf/layer.h"
Expand All @@ -18,10 +19,18 @@
#include <fstream>
#include <iostream>
#include <map>

#include <mutex>
#include <thread>
#include <string>
#include <regex>

/*
Safety-wise we lock via a mutex when we re-rout the relative path lookup to a Python call.
In theory we probably don't need this, as our Python call does this anyway.
See the _CreateIdentifier method for more information.
*/
static std::mutex g_resolver_create_identifier_mutex;

namespace python = AR_BOOST_NAMESPACE::python;

PXR_NAMESPACE_OPEN_SCOPE
Expand Down Expand Up @@ -80,6 +89,38 @@ CachedResolver::CachedResolver() = default;

CachedResolver::~CachedResolver() = default;

void CachedResolver::AddCachedRelativePathIdentifierPair(const std::string& sourceStr, const std::string& targetStr){
auto cache_find = cachedRelativePathIdentifierPairs.find(sourceStr);
if(cache_find != cachedRelativePathIdentifierPairs.end()){
cache_find->second = targetStr;
}else{
cachedRelativePathIdentifierPairs.insert(std::pair<std::string, std::string>(sourceStr,targetStr));
}
}


void CachedResolver::RemoveCachedRelativePathIdentifierByKey(const std::string& sourceStr){
const auto &it = cachedRelativePathIdentifierPairs.find(sourceStr);
if (it != cachedRelativePathIdentifierPairs.end()){
cachedRelativePathIdentifierPairs.erase(it);
}
}


void CachedResolver::RemoveCachedRelativePathIdentifierByValue(const std::string& targetStr){
for (auto it = cachedRelativePathIdentifierPairs.cbegin(); it != cachedRelativePathIdentifierPairs.cend();)
{
if (it->second == targetStr)
{
cachedRelativePathIdentifierPairs.erase(it++);
}
else
{
++it;
}
}
}

std::string
CachedResolver::_CreateIdentifier(
const std::string& assetPath,
Expand All @@ -97,11 +138,46 @@ CachedResolver::_CreateIdentifier(
}

const std::string anchoredAssetPath = _AnchorRelativePath(anchorAssetPath, assetPath);

// Anchor non file path based identifiers and see if a file exists.
// This is mostly for debugging as it allows us to add a file relative to our
// anchor directory that has a higher priority than our (usually unanchored)
// resolved asset path.
if (_IsNotFilePath(assetPath) && Resolve(anchoredAssetPath).empty()) {
return TfNormPath(assetPath);
}

// Re-direct to Python to allow optional re-routing of relative paths
// through the resolver.
if (true) {
if (_IsFileRelativePath(assetPath)) {
auto cache_find = this->cachedRelativePathIdentifierPairs.find(anchorAssetPath);
if(cache_find != this->cachedRelativePathIdentifierPairs.end()){
return cache_find->second;
}else{
std::string pythonResult;
{
/*
We optionally re-route relative file paths to be pre-formatted in Python.
This allows us to optionally override relative paths too and not only
non file path identifiers.
Please see the CachedResolverContext::ResolveAndCachePair method for
risks (as this does the same hacky workaround)
and the Python Resolver.CreateRelativePathIdentified method on how to use this.
*/
const std::lock_guard<std::mutex> lock(g_resolver_create_identifier_mutex);
int state = TfPyInvokeAndExtract(DEFINE_STRING(AR_CACHEDRESOLVER_USD_PYTHON_EXPOSE_MODULE_NAME),
"Resolver.CreateRelativePathIdentifier",
&pythonResult, AR_BOOST_NAMESPACE::ref(*this), anchoredAssetPath, assetPath, anchorAssetPath);
if (!state) {
std::cerr << "Failed to call Resolver.CreateRelativePathIdentifier in " << DEFINE_STRING(AR_CACHEDRESOLVER_USD_PYTHON_EXPOSE_MODULE_NAME) << ".py. ";
std::cerr << "Please verify that the python code is valid!" << std::endl;
pythonResult = TfNormPath(anchoredAssetPath);
}
}
return pythonResult;
}
}
}
return TfNormPath(anchoredAssetPath);
}

Expand Down Expand Up @@ -137,51 +213,51 @@ CachedResolver::_Resolve(
if (SdfLayer::IsAnonymousLayerIdentifier(assetPath)){
return ArResolvedPath(assetPath);
}
if (_IsRelativePath(assetPath)) {
if (this->_IsContextDependentPath(assetPath)) {
const CachedResolverContext* contexts[2] = {this->_GetCurrentContextPtr(), &_fallbackContext};
for (const CachedResolverContext* ctx : contexts) {
if (ctx) {
// Search for mapping pairs
auto &mappingPairs = ctx->GetMappingPairs();
auto map_find = mappingPairs.find(assetPath);
if(map_find != mappingPairs.end()){
ArResolvedPath resolvedPath = _ResolveAnchored(this->emptyString, map_find->second);
return resolvedPath;
// Assume that a map hit is always valid.
// if (resolvedPath) {
// return resolvedPath;
// }
}
// Search for cached pairs
auto &cachedPairs = ctx->GetCachingPairs();
auto cache_find = cachedPairs.find(assetPath);
if(cache_find != cachedPairs.end()){
ArResolvedPath resolvedPath = _ResolveAnchored(this->emptyString, cache_find->second);
return resolvedPath;
// Assume that a cache hit is always valid.
// if (resolvedPath) {
// return resolvedPath;
// }
}
// Perform query if caches don't have a hit.
TF_DEBUG(CACHEDRESOLVER_RESOLVER).Msg("Resolver::_Resolve('%s') -> No cache hit, switching to Python query\n", assetPath.c_str());
/*
We perform the resource/thread lock in the context itself to
allow for resolver multithreading with different contexts.
See .ResolveAndCachePair for more information.
*/
ArResolvedPath resolvedPath = _ResolveAnchored(this->emptyString, ctx->ResolveAndCachePair(assetPath));
if (resolvedPath) {
return resolvedPath;
}
// Only try the first valid context.
break;

if (this->_IsContextDependentPath(assetPath)) {
const CachedResolverContext* contexts[2] = {this->_GetCurrentContextPtr(), &_fallbackContext};
for (const CachedResolverContext* ctx : contexts) {
if (ctx) {
// Search for mapping pairs
auto &mappingPairs = ctx->GetMappingPairs();
auto map_find = mappingPairs.find(assetPath);
if(map_find != mappingPairs.end()){
ArResolvedPath resolvedPath = _ResolveAnchored(this->emptyString, map_find->second);
return resolvedPath;
// Assume that a map hit is always valid.
// if (resolvedPath) {
// return resolvedPath;
// }
}
// Search for cached pairs
auto &cachedPairs = ctx->GetCachingPairs();
auto cache_find = cachedPairs.find(assetPath);
if(cache_find != cachedPairs.end()){
ArResolvedPath resolvedPath = _ResolveAnchored(this->emptyString, cache_find->second);
return resolvedPath;
// Assume that a cache hit is always valid.
// if (resolvedPath) {
// return resolvedPath;
// }
}
// Perform query if caches don't have a hit.
TF_DEBUG(CACHEDRESOLVER_RESOLVER).Msg("Resolver::_Resolve('%s') -> No cache hit, switching to Python query\n", assetPath.c_str());
/*
We perform the resource/thread lock in the context itself to
allow for resolver multithreading with different contexts.
See .ResolveAndCachePair for more information.
*/
ArResolvedPath resolvedPath = _ResolveAnchored(this->emptyString, ctx->ResolveAndCachePair(assetPath));
if (resolvedPath) {
return resolvedPath;
}
// Only try the first valid context.
break;
}
}
return ArResolvedPath();
}

return _ResolveAnchored(std::string(), assetPath);
}

Expand Down
22 changes: 11 additions & 11 deletions src/CachedResolver/resolver.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,53 +27,52 @@ class CachedResolver final : public ArResolver
public:
AR_CACHEDRESOLVER_API
CachedResolver();

AR_CACHEDRESOLVER_API
virtual ~CachedResolver();

AR_CACHEDRESOLVER_API
void AddCachedRelativePathIdentifierPair(const std::string& sourceStr, const std::string& targetStr);
AR_CACHEDRESOLVER_API
void RemoveCachedRelativePathIdentifierByKey(const std::string& sourceStr);
AR_CACHEDRESOLVER_API
void RemoveCachedRelativePathIdentifierByValue(const std::string& targetStr);
AR_CACHEDRESOLVER_API
const std::map<std::string, std::string>& GetCachedRelativePathIdentifierPairs() const { return cachedRelativePathIdentifierPairs; }
AR_CACHEDRESOLVER_API
void ClearCachedRelativePathIdentifierPairs() { cachedRelativePathIdentifierPairs.clear(); }
protected:
AR_CACHEDRESOLVER_API
std::string _CreateIdentifier(
const std::string& assetPath,
const ArResolvedPath& anchorAssetPath) const final;

AR_CACHEDRESOLVER_API
std::string _CreateIdentifierForNewAsset(
const std::string& assetPath,
const ArResolvedPath& anchorAssetPath) const final;

AR_CACHEDRESOLVER_API
ArResolvedPath _Resolve(
const std::string& assetPath) const final;

AR_CACHEDRESOLVER_API
ArResolvedPath _ResolveForNewAsset(
const std::string& assetPath) const final;

AR_CACHEDRESOLVER_API
ArResolverContext _CreateDefaultContext() const final;

AR_CACHEDRESOLVER_API
ArResolverContext _CreateDefaultContextForAsset(
const std::string& assetPath) const final;

AR_CACHEDRESOLVER_API
bool _IsContextDependentPath(
const std::string& assetPath) const final;

AR_CACHEDRESOLVER_API
void _RefreshContext(
const ArResolverContext& context) final;

AR_CACHEDRESOLVER_API
ArTimestamp _GetModificationTimestamp(
const std::string& assetPath,
const ArResolvedPath& resolvedPath) const final;

AR_CACHEDRESOLVER_API
std::shared_ptr<ArAsset> _OpenAsset(
const ArResolvedPath& resolvedPath) const final;

AR_CACHEDRESOLVER_API
std::shared_ptr<ArWritableAsset> _OpenAssetForWrite(
const ArResolvedPath& resolvedPath,
Expand All @@ -83,6 +82,7 @@ class CachedResolver final : public ArResolver
const CachedResolverContext* _GetCurrentContextPtr() const;
CachedResolverContext _fallbackContext;
const std::string emptyString{};
std::map<std::string, std::string> cachedRelativePathIdentifierPairs;
};

PXR_NAMESPACE_CLOSE_SCOPE
Expand Down
Loading

0 comments on commit 5c53805

Please sign in to comment.