From 9fc5970a55ccea1c3894fc1bdf7004a5e600bf2e Mon Sep 17 00:00:00 2001 From: Luca Scheller Date: Fri, 24 Nov 2023 20:50:35 +0100 Subject: [PATCH] Update docs about relative identifier exposure --- docs/src/SUMMARY.md | 3 +- .../src/resolvers/CachedResolver/PythonAPI.md | 77 ++++++++++++++++++- docs/src/resolvers/CachedResolver/example.md | 1 + docs/src/resolvers/CachedResolver/overview.md | 9 ++- docs/src/resolvers/ExampleSetup/overview.md | 14 ++-- files/{ => generic}/box.usda | 0 files/{ => generic}/cylinder.usda | 0 files/{ => generic}/mapping.usda | 0 .../workspace/assets/assetA/assetA.usd | 0 .../workspace/assets/assetA/assetA_v001.usd | 0 .../workspace/assets/assetA/assetA_v002.usd | 0 .../workspace/shots/shotA/shotA.usd | 0 .../workspace/shots/shotA/shotA_mapping.usd | 0 .../workspace/assets/assetA/assetA.usd | 5 ++ .../workspace/assets/assetA/assetA_v001.usd | 5 ++ .../workspace/assets/assetA/assetA_v002.usd | 4 + .../workspace/shots/shotA/shotA.usd | 6 ++ .../workspace/shots/shotA/shotA_mapping.usd | 6 ++ setup.sh | 2 +- src/CachedResolver/PythonExpose.py | 13 ++-- src/CachedResolver/wrapResolver.cpp | 6 +- 21 files changed, 131 insertions(+), 20 deletions(-) create mode 100644 docs/src/resolvers/CachedResolver/example.md rename files/{ => generic}/box.usda (100%) rename files/{ => generic}/cylinder.usda (100%) rename files/{ => generic}/mapping.usda (100%) rename files/{ => generic}/workspace/assets/assetA/assetA.usd (100%) rename files/{ => generic}/workspace/assets/assetA/assetA_v001.usd (100%) rename files/{ => generic}/workspace/assets/assetA/assetA_v002.usd (100%) rename files/{ => generic}/workspace/shots/shotA/shotA.usd (100%) rename files/{ => generic}/workspace/shots/shotA/shotA_mapping.usd (100%) create mode 100644 files/implementations/CachedResolver/workspace/assets/assetA/assetA.usd create mode 100644 files/implementations/CachedResolver/workspace/assets/assetA/assetA_v001.usd create mode 100644 files/implementations/CachedResolver/workspace/assets/assetA/assetA_v002.usd create mode 100644 files/implementations/CachedResolver/workspace/shots/shotA/shotA.usd create mode 100644 files/implementations/CachedResolver/workspace/shots/shotA/shotA_mapping.usd diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index a0925fe..86a35a0 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -17,9 +17,10 @@ - [Python API](./resolvers/FileResolver/PythonAPI.md) - [Cached Resolver](./resolvers/CachedResolver/overview.md) - [Python API](./resolvers/CachedResolver/PythonAPI.md) + - [Production Setup](./resolvers/CachedResolver/example.md) - [RnD Resolvers](./resolvers/rnd.md) - [Python Resolver](./resolvers/PythonResolver/overview.md) - [Python API](./resolvers/PythonResolver/PythonAPI.md) - [Proof Of Concept Resolvers](./resolvers/proofofconcept.md) - [Http Resolver ('arHttp' by @charlesfleche)](./resolvers/HttpResolver/overview.md) - - [Example Setup](./resolvers/ExampleSetup/overview.md) \ No newline at end of file + - [Example Setup (File/PythonResolver)](./resolvers/ExampleSetup/overview.md) \ No newline at end of file diff --git a/docs/src/resolvers/CachedResolver/PythonAPI.md b/docs/src/resolvers/CachedResolver/PythonAPI.md index 03736bb..621e0b3 100644 --- a/docs/src/resolvers/CachedResolver/PythonAPI.md +++ b/docs/src/resolvers/CachedResolver/PythonAPI.md @@ -11,6 +11,44 @@ Tokens can be found in CachedResolver.Tokens: CachedResolver.Tokens.mappingPairs ``` +## Resolver +We optionally can also hook into relative path identifier creation via Python. + +This can be enabled by setting the `AR_CACHEDRESOLVER_ENV_EXPOSE_RELATIVE_PATH_IDENTIFIERS` environment variable to `1` or by calling `pxr.Ar.GetUnderlyingResolver().SetExposeRelativePathIdentifierState(True)`. + +We then have access in our `PythonExpose.py` -> `Resolver.CreateRelativePathIdentifier` method. Here we can then return a non file path (anything that doesn't start with "/"/"./"/"../") identifier for our relative path, which then also gets passed to our `PythonExpose.py` -> `ResolverContext.ResolveAndCache` method. + +This allows us to also redirect relative paths to our liking for example when implementing special pinning/mapping behaviours. + +For more info check out our [production example](./example.md) section. + +As with our mapping and caching pairs, the result is cached in C++ to enable faster lookups on consecutive calls. + +As identifiers are context independent, the cache is stored on the resolver itself. See below on how to modify and inspect the cache. + +```python +from pxr import Ar, Usd +from usdAssetResolver import CachedResolver + +# Enable relative identifier modification +cached_resolver = Ar.GetUnderlyingResolver() +cached_resolver.SetExposeRelativePathIdentifierState(True) +print("Resolver is currently exposing relative path identifiers to Python | {}".format(cached_resolver.GetExposeRelativePathIdentifierState())) +# Or set the "AR_CACHEDRESOLVER_ENV_EXPOSE_RELATIVE_PATH_IDENTIFIERS" environment variable to 1. +# This can't be done via Python, as it has to happen before the resolver is loaded. +cached_resolver.GetExposeRelativePathIdentifierState() # Get the state of exposing relative path identifiers +cached_resolver.SetExposeRelativePathIdentifierState() # Set the state of exposing relative path identifiers + +# We can also inspect and edit our relative identifiers via the following methods on the resolver. +# You'll usually only want to use these outside of the Resolver.CreateRelativePathIdentifier method only for debugging. +# The identifier cache is not context dependent (as identifiers are not). +cached_resolver.GetCachedRelativePathIdentifierPairs() # Returns all cached relative path identifier pairs as a dict +cached_resolver.AddCachedRelativePathIdentifierPair() # Add a cached relative path identifier pair +cached_resolver.RemoveCachedRelativePathIdentifierByKey() # Remove a cached relative path identifier pair by key +cached_resolver.RemoveCachedRelativePathIdentifierByValue() # Remove a cached relative path identifier pair by value +cached_resolver.ClearCachedRelativePathIdentifierPairs() # Clear all cached relative path identifier pairs +``` + ## Resolver Context You can manipulate the resolver context (the object that holds the configuration the resolver uses to resolve paths) via Python in the following ways: @@ -136,6 +174,41 @@ def log_function_args(func): ...code... ``` +#### Resolver + +```python +class Resolver: + + @staticmethod + @log_function_args + def CreateRelativePathIdentifier(resolver, anchoredAssetPath, assetPath, anchorAssetPath): + """Returns an identifier for the asset specified by assetPath. + It is very important that the anchoredAssetPath is used as the cache key, as this + is what is used in C++ to do the cache lookup. + + We have two options how to return relative identifiers: + - Make it absolute: Simply return the anchoredAssetPath. This means the relative identifier + will not be passed through to ResolverContext.ResolveAndCache. + - Make it non file based: Make sure the remapped identifier does not start with "/", "./" or"../" + by putting some sort of prefix in front of it. The path will then be + passed through to ResolverContext.ResolveAndCache, where you need to re-construct + it to an absolute path of your liking. Make sure you don't use a ":", + to avoid mixups with URI based resolvers. + + Args: + resolver (CachedResolver): The resolver + anchoredAssetPath (str): The anchored asset path, this has to be used as the cached key. + assetPath (str): An unresolved asset path. + anchorAssetPath (Ar.ResolvedPath): A resolved anchor path. + + Returns: + str: The identifier. + """ + remappedRelativePathIdentifier = f"relativePath|{assetPath}?{anchorAssetPath}" + resolver.AddCachedRelativePathIdentifierPair(anchoredAssetPath, remappedRelativePathIdentifier) + return remappedRelativePathIdentifier +``` + #### Resolver Context ```python @@ -158,12 +231,12 @@ class ResolverContext: context.AddCachingPair("identifier", "/some/path/to/a/file.usd") @staticmethod - def ResolveAndCache(assetPath, context): + def ResolveAndCache(context, assetPath): """Return the resolved path for the given assetPath or an empty ArResolvedPath if no asset exists at that path. Args: - assetPath (str): An unresolved asset path. context (CachedResolverContext): The active context. + assetPath (str): An unresolved asset path. Returns: str: The resolved path string. If it points to a non-existent file, it will be resolved to an empty ArResolvedPath internally, but will diff --git a/docs/src/resolvers/CachedResolver/example.md b/docs/src/resolvers/CachedResolver/example.md new file mode 100644 index 0000000..342bfd5 --- /dev/null +++ b/docs/src/resolvers/CachedResolver/example.md @@ -0,0 +1 @@ +# Production Setup diff --git a/docs/src/resolvers/CachedResolver/overview.md b/docs/src/resolvers/CachedResolver/overview.md index 37327b6..00990b4 100644 --- a/docs/src/resolvers/CachedResolver/overview.md +++ b/docs/src/resolvers/CachedResolver/overview.md @@ -4,7 +4,11 @@ This resolver first consults an internal resolver context dependent cache to resolve asset paths. If the asset path is not found in the cache, it will redirect the request to Python and cache the result. This is ideal for smaller studios, as this preserves the speed of C++ with the flexibility of Python. ``` -Similar to the FileResolver and USD's default resolver, any absolute and relative file path is resolved as an on-disk file path. That means "normal" USD files, that don't use custom identifiers, will resolve as expected (and as fast as usual as this is called in C++). +By default (similar to the FileResolver and USD's default resolver), any absolute and relative file path is resolved as an on-disk file path. That means "normal" USD files, that don't use custom identifiers, will resolve as expected (and as fast as usual as this is called in C++). + +```admonish tip title="Pro Tip" +Optionally you can opt-in into also exposing relative identifiers to Python by setting the `AR_CACHEDRESOLVER_ENV_EXPOSE_RELATIVE_PATH_IDENTIFIERS` environment variable to `1` or by calling `pxr.Ar.GetUnderlyingResolver().SetExposeRelativePathIdentifierState(True)`. This is a more advanced feature and is therefore disabled by default. See our [production example](example.md) section for more information on how to use this and why it can be useful. +``` All non file path identifiers (anything that doesn't start with "/"/"./"/"../") will forward their request to the `PythonExpose.py` -> `ResolverContext.ResolveAndCache` method. If you want to customize this resolver, just edit the methods in PythonExpose.py to fit your needs. You can either edit the file directly or move it anywhere where your "PYTHONPATH"/"sys.path" paths look for Python modules. @@ -17,7 +21,8 @@ Here is a full list of features: - **CachingPairs**: All resolver context methods that have `Caching` in their name, modify the internal `cachingPairs` dictionary. With this dictionary it is up to you when to populate it. In our `PythonExpose.py` file, we offer two ways where you can hook into the resolve process. In both of them you can add as many cached lookups as you want via `ctx.AddCachingPair(asset_path, resolved_asset_path)`: - On context creation via the `PythonExpose.py` -> `ResolverContext.Initialize` method. This gets called whenever a context gets created (including the fallback default context). For example Houdini creates the default context if you didn't specify a "Resolver Context Asset Path" in your stage on the active node/in the stage network. If you do specify one, then a new context gets spawned that does the above mentioned mapping pair lookup and then runs the `PythonExpose.py` -> `ResolverContext.Initialize` method. - On resolve for non file path identifiers (anything that doesn't start with "/"/"./"/"../") via the `PythonExpose.py` -> `ResolverContext.ResolveAndCache` method. Here you are free to only add the active asset path via `ctx.AddCachingPair(asset_path, resolved_asset_path)` or any number of relevant asset paths. -- In comparison to our [FileResolver](../FileResolver/overview.md) and [PythonResolver](../PythonResolver/overview.md), the mapping/caching pair values need to point to the absolute disk path. We chose to make this behavior different, because in the "PythonExpose.py" you can directly customize the "final" on-disk path to your liking. +- We optionally also support hooking into relative path identifier creation via Python. This can be enabled by setting the `AR_CACHEDRESOLVER_ENV_EXPOSE_RELATIVE_PATH_IDENTIFIERS` environment variable to `1` or by calling `pxr.Ar.GetUnderlyingResolver().SetExposeRelativePathIdentifierState(True)`. We then have access in our `PythonExpose.py` -> `Resolver.CreateRelativePathIdentifier` method. Here we can then return a non file path (anything that doesn't start with "/"/"./"/"../") identifier for our relative path, which then also gets passed to our `PythonExpose.py` -> `ResolverContext.ResolveAndCache` method. This allows us to also redirect relative paths to our liking for example when implementing special pinning/mapping behaviours. For more info check out our [production example](./example.md) section. As with our mapping and caching pairs, the result is cached in C++ to enable faster lookups on consecutive calls. As identifiers are context independent, the cache is stored on the resolver itself. See our [Python API](./PythonAPI.md) section on how to clear the cache. +- In comparison to our [FileResolver](../FileResolver/overview.md) and [PythonResolver](../PythonResolver/overview.md), the mapping/caching pair values need to point to the absolute disk path (instead of using a search path). We chose to make this behavior different, because in the "PythonExpose.py" you can directly customize the "final" on-disk path to your liking. - The resolver contexts are cached globally, so that DCCs, that try to spawn a new context based on the same mapping file using the [```Resolver.CreateDefaultContextForAsset```](https://openusd.org/dev/api/class_ar_resolver.html), will re-use the same cached resolver context. The resolver context cache key is currently the mapping file path. This may be subject to change, as a hash might be a good alternative, as it could also cover non file based edits via the exposed Python resolver API. - ```Resolver.CreateContextFromString```/```Resolver.CreateContextFromStrings``` is not implemented due to many DCCs not making use of it yet. As we expose the ability to edit the context at runtime, this is also often not necessary. If needed please create a request by submitting an issue here: [Create New Issue](https://github.com/LucaScheller/VFX-UsdAssetResolver/issues/new) - Refreshing the stage is also supported, although it might be required to trigger additional reloads in certain DCCs. diff --git a/docs/src/resolvers/ExampleSetup/overview.md b/docs/src/resolvers/ExampleSetup/overview.md index e2b6db8..97fa358 100644 --- a/docs/src/resolvers/ExampleSetup/overview.md +++ b/docs/src/resolvers/ExampleSetup/overview.md @@ -1,16 +1,18 @@ # Example Usd files and mapping pair files +```admonish warning +These examples currently work with the [File Resolver](../FileResolver/overview.md) and [Python Resolver](../PythonResolver/overview.md). + +For resolver specific examples, see the corresponding resolver section. +``` + ## Simple Example A very simple setup can be found in In the /files folder. -```admonish info -These examples currently work with the [File Resolver](../FileResolver/overview.md) and [Python Resolver](../PythonResolver/overview.md) -``` - Before launching a Usd related software, you'll have to set these env vars: ```bash -export AR_SEARCH_PATHS=${REPO_ROOT}/files +export AR_SEARCH_PATHS=${REPO_ROOT}/files/generic export AR_SEARCH_REGEX_EXPRESSION="(bo)" export AR_SEARCH_REGEX_FORMAT="Bo" ``` @@ -31,7 +33,7 @@ A larger example scene setup might looks as follows: In the /files folder you can also find this setup. To run it, you must set the `AR_SEARCH_PATHS` env var as follows. ```bash -export AR_SEARCH_PATHS=${REPO_ROOT}/files/workspace/shots:${REPO_ROOT}/files/workspace/assets +export AR_SEARCH_PATHS=${REPO_ROOT}/files/generic/workspace/shots:${REPO_ROOT}/files/generic/workspace/assets ``` And then open up the `shots/shotA/shotA.usd` file and set the resolver context mapping file path to `shots/shotA/shotA_mapping.usd`. diff --git a/files/box.usda b/files/generic/box.usda similarity index 100% rename from files/box.usda rename to files/generic/box.usda diff --git a/files/cylinder.usda b/files/generic/cylinder.usda similarity index 100% rename from files/cylinder.usda rename to files/generic/cylinder.usda diff --git a/files/mapping.usda b/files/generic/mapping.usda similarity index 100% rename from files/mapping.usda rename to files/generic/mapping.usda diff --git a/files/workspace/assets/assetA/assetA.usd b/files/generic/workspace/assets/assetA/assetA.usd similarity index 100% rename from files/workspace/assets/assetA/assetA.usd rename to files/generic/workspace/assets/assetA/assetA.usd diff --git a/files/workspace/assets/assetA/assetA_v001.usd b/files/generic/workspace/assets/assetA/assetA_v001.usd similarity index 100% rename from files/workspace/assets/assetA/assetA_v001.usd rename to files/generic/workspace/assets/assetA/assetA_v001.usd diff --git a/files/workspace/assets/assetA/assetA_v002.usd b/files/generic/workspace/assets/assetA/assetA_v002.usd similarity index 100% rename from files/workspace/assets/assetA/assetA_v002.usd rename to files/generic/workspace/assets/assetA/assetA_v002.usd diff --git a/files/workspace/shots/shotA/shotA.usd b/files/generic/workspace/shots/shotA/shotA.usd similarity index 100% rename from files/workspace/shots/shotA/shotA.usd rename to files/generic/workspace/shots/shotA/shotA.usd diff --git a/files/workspace/shots/shotA/shotA_mapping.usd b/files/generic/workspace/shots/shotA/shotA_mapping.usd similarity index 100% rename from files/workspace/shots/shotA/shotA_mapping.usd rename to files/generic/workspace/shots/shotA/shotA_mapping.usd diff --git a/files/implementations/CachedResolver/workspace/assets/assetA/assetA.usd b/files/implementations/CachedResolver/workspace/assets/assetA/assetA.usd new file mode 100644 index 0000000..a136181 --- /dev/null +++ b/files/implementations/CachedResolver/workspace/assets/assetA/assetA.usd @@ -0,0 +1,5 @@ +#usda 1.0 +def Cube "asset" () +{ + double size = 2 +} \ No newline at end of file diff --git a/files/implementations/CachedResolver/workspace/assets/assetA/assetA_v001.usd b/files/implementations/CachedResolver/workspace/assets/assetA/assetA_v001.usd new file mode 100644 index 0000000..a136181 --- /dev/null +++ b/files/implementations/CachedResolver/workspace/assets/assetA/assetA_v001.usd @@ -0,0 +1,5 @@ +#usda 1.0 +def Cube "asset" () +{ + double size = 2 +} \ No newline at end of file diff --git a/files/implementations/CachedResolver/workspace/assets/assetA/assetA_v002.usd b/files/implementations/CachedResolver/workspace/assets/assetA/assetA_v002.usd new file mode 100644 index 0000000..63fbef7 --- /dev/null +++ b/files/implementations/CachedResolver/workspace/assets/assetA/assetA_v002.usd @@ -0,0 +1,4 @@ +#usda 1.0 +def Cylinder "asset" () +{ +} \ No newline at end of file diff --git a/files/implementations/CachedResolver/workspace/shots/shotA/shotA.usd b/files/implementations/CachedResolver/workspace/shots/shotA/shotA.usd new file mode 100644 index 0000000..a35e6ff --- /dev/null +++ b/files/implementations/CachedResolver/workspace/shots/shotA/shotA.usd @@ -0,0 +1,6 @@ +#usda 1.0 +def "testAssetA" ( + prepend references = @assetA/assetA.usd@ +) +{ +} \ No newline at end of file diff --git a/files/implementations/CachedResolver/workspace/shots/shotA/shotA_mapping.usd b/files/implementations/CachedResolver/workspace/shots/shotA/shotA_mapping.usd new file mode 100644 index 0000000..9eee1a5 --- /dev/null +++ b/files/implementations/CachedResolver/workspace/shots/shotA/shotA_mapping.usd @@ -0,0 +1,6 @@ +#usda 1.0 +( + customLayerData = { + string[] mappingPairs = ["assetA/assetA.usd", "assetA/assetA_v002.usd"] + } +) \ No newline at end of file diff --git a/setup.sh b/setup.sh index 17585df..4e7343b 100644 --- a/setup.sh +++ b/setup.sh @@ -18,7 +18,7 @@ then export LD_LIBRARY_PATH=${REPO_ROOT}/dist/${RESOLVER_NAME}/lib:${LD_LIBRARY_PATH} alias usdpython="$HFS/python/bin/python $@" # Configure resolver - export AR_SEARCH_PATHS=${REPO_ROOT}/files + export AR_SEARCH_PATHS=${REPO_ROOT}/files/generic export AR_SEARCH_REGEX_EXPRESSION="(bo)" export AR_SEARCH_REGEX_FORMAT="Bo" # Debug diff --git a/src/CachedResolver/PythonExpose.py b/src/CachedResolver/PythonExpose.py index c268fbe..164830c 100644 --- a/src/CachedResolver/PythonExpose.py +++ b/src/CachedResolver/PythonExpose.py @@ -45,15 +45,18 @@ class Resolver: @staticmethod @log_function_args def CreateRelativePathIdentifier(resolver, anchoredAssetPath, assetPath, anchorAssetPath): - """Returns an identifier for the asset specified by assetPath. + """Returns an identifier for the asset specified by assetPath and anchor asset path. + It is very important that the anchoredAssetPath is used as the cache key, as this + is what is used in C++ to do the cache lookup. We have two options how to return relative identifiers: - Make it absolute: Simply return the anchoredAssetPath. This means the relative identifier - will not be passed through to ResolveAndCache. - - Make it non file based: Make sure the remapped identifier does not start with "./" or "../" + will not be passed through to ResolverContext.ResolveAndCache. + - Make it non file based: Make sure the remapped identifier does not start with "/", "./" or"../" by putting some sort of prefix in front of it. The path will then be - passed through to ResolveAndCache, where you need to re-construct - it to an absolute path of your liking. + passed through to ResolverContext.ResolveAndCache, where you need to re-construct + it to an absolute path of your liking. Make sure you don't use a ":", + to avoid mixups with URI based resolvers. Args: resolver (CachedResolver): The resolver diff --git a/src/CachedResolver/wrapResolver.cpp b/src/CachedResolver/wrapResolver.cpp index 9b5f2a1..8dbedad 100644 --- a/src/CachedResolver/wrapResolver.cpp +++ b/src/CachedResolver/wrapResolver.cpp @@ -21,9 +21,9 @@ wrapResolver() .def("GetExposeRelativePathIdentifierState", &This::GetExposeRelativePathIdentifierState, return_value_policy(), "Get the state of exposing relative path identifiers") .def("SetExposeRelativePathIdentifierState", &This::SetExposeRelativePathIdentifierState, "Set the state of exposing relative path identifiers") .def("GetCachedRelativePathIdentifierPairs", &This::GetCachedRelativePathIdentifierPairs, return_value_policy(), "Returns all cached relative path identifier pairs as a dict") - .def("AddCachedRelativePathIdentifierPair", &This::AddCachedRelativePathIdentifierPair, "Remove a cached relative path identifier pair by value") - .def("RemoveCachedRelativePathIdentifierByKey", &This::RemoveCachedRelativePathIdentifierByKey, "Add a cached relative path identifier pair") - .def("RemoveCachedRelativePathIdentifierByValue", &This::RemoveCachedRelativePathIdentifierByValue, "Remove a cached relative path identifier pair by key") + .def("AddCachedRelativePathIdentifierPair", &This::AddCachedRelativePathIdentifierPair, "Add a cached relative path identifier pair") + .def("RemoveCachedRelativePathIdentifierByKey", &This::RemoveCachedRelativePathIdentifierByKey, "Remove a cached relative path identifier pair by key") + .def("RemoveCachedRelativePathIdentifierByValue", &This::RemoveCachedRelativePathIdentifierByValue, "Remove a cached relative path identifier pair by value") .def("ClearCachedRelativePathIdentifierPairs", &This::ClearCachedRelativePathIdentifierPairs, "Clear all cached relative path identifier pairs") ; } \ No newline at end of file