From e077ff1f38dd89eba881243f194d276a15c7804a Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Mon, 19 Aug 2024 13:54:36 +0200 Subject: [PATCH] src: update compile cache storage structure This refactors the compile cache handler in preparation for the JS API, and updates the compile cache storage structure into: - $NODE_COMPILE_CACHE_DIR - $NODE_VERION-$ARCH-$CACHE_DATA_VERSION_TAG-$UID - $FILENAME_AND_MODULE_TYPE_HASH.cache This also adds a magic number to the beginning of the cache files for verification, and returns the status, compile cache directory and/or error message of enabling the compile cache in a structure, which can be converted as JS counterparts by the upcoming JS API. PR-URL: https://github.com/nodejs/node/pull/54291 Refs: https://github.com/nodejs/node/issues/53639 Reviewed-By: Anna Henningsen Reviewed-By: James M Snell Reviewed-By: Ethan Arrowood Reviewed-By: Chengzhong Wu --- src/compile_cache.cc | 117 +++++++++++------- src/compile_cache.h | 36 ++++-- src/env.cc | 37 ++++-- src/env.h | 3 + ...est-compile-cache-permission-disallowed.js | 6 +- 5 files changed, 137 insertions(+), 62 deletions(-) diff --git a/src/compile_cache.cc b/src/compile_cache.cc index 5c925469bd203f..e7940c85e9057c 100644 --- a/src/compile_cache.cc +++ b/src/compile_cache.cc @@ -1,4 +1,5 @@ #include "compile_cache.h" +#include #include "debug_utils-inl.h" #include "env-inl.h" #include "node_file.h" @@ -27,15 +28,19 @@ uint32_t GetHash(const char* data, size_t size) { return crc32(crc, reinterpret_cast(data), size); } -uint32_t GetCacheVersionTag() { - std::string_view node_version(NODE_VERSION); - uint32_t v8_tag = v8::ScriptCompiler::CachedDataVersionTag(); - uLong crc = crc32(0L, Z_NULL, 0); - crc = crc32(crc, reinterpret_cast(&v8_tag), sizeof(uint32_t)); - crc = crc32(crc, - reinterpret_cast(node_version.data()), - node_version.size()); - return crc; +std::string GetCacheVersionTag() { + // On platforms where uids are available, use different folders for + // different users to avoid cache miss due to permission incompatibility. + // On platforms where uids are not available, bare with the cache miss. + // This should be fine on Windows, as there local directories tend to be + // user-specific. + std::string tag = std::string(NODE_VERSION) + '-' + std::string(NODE_ARCH) + + '-' + + Uint32ToHex(v8::ScriptCompiler::CachedDataVersionTag()); +#ifdef NODE_IMPLEMENTS_POSIX_CREDENTIALS + tag += '-' + std::to_string(getuid()); +#endif + return tag; } uint32_t GetCacheKey(std::string_view filename, CachedCodeType type) { @@ -63,6 +68,10 @@ v8::ScriptCompiler::CachedData* CompileCacheEntry::CopyCache() const { data, cache_size, v8::ScriptCompiler::CachedData::BufferOwned); } +// Used for identifying and verifying a file is a compile cache file. +// See comments in CompileCacheHandler::Persist(). +constexpr uint32_t kCacheMagicNumber = 0x8adfdbb2; + void CompileCacheHandler::ReadCacheFile(CompileCacheEntry* entry) { Debug("[compile cache] reading cache from %s for %s %s...", entry->cache_filename, @@ -100,12 +109,20 @@ void CompileCacheHandler::ReadCacheFile(CompileCacheEntry* entry) { return; } - Debug("[%d %d %d %d]...", + Debug("[%d %d %d %d %d]...", + headers[kMagicNumberOffset], headers[kCodeSizeOffset], headers[kCacheSizeOffset], headers[kCodeHashOffset], headers[kCacheHashOffset]); + if (headers[kMagicNumberOffset] != kCacheMagicNumber) { + Debug("magic number mismatch: expected %d, actual %d\n", + kCacheMagicNumber, + headers[kMagicNumberOffset]); + return; + } + // Check the code size and hash which are already computed. if (headers[kCodeSizeOffset] != entry->code_size) { Debug("code size mismatch: expected %d, actual %d\n", @@ -202,11 +219,14 @@ CompileCacheEntry* CompileCacheHandler::GetOrInsert( compiler_cache_store_.emplace(key, std::make_unique()); auto* result = emplaced.first->second.get(); + std::u8string cache_filename_u8 = + (compile_cache_dir_ / Uint32ToHex(key)).u8string(); result->code_hash = code_hash; result->code_size = code_utf8.length(); result->cache_key = key; result->cache_filename = - (compile_cache_dir_ / Uint32ToHex(result->cache_key)).string(); + std::string(cache_filename_u8.begin(), cache_filename_u8.end()) + + ".cache"; result->source_filename = filename_utf8.ToString(); result->cache = nullptr; result->type = type; @@ -264,6 +284,7 @@ void CompileCacheHandler::MaybeSave(CompileCacheEntry* entry, } // Layout of a cache file: +// [uint32_t] magic number // [uint32_t] code size // [uint32_t] code hash // [uint32_t] cache size @@ -301,14 +322,16 @@ void CompileCacheHandler::Persist() { // Generating headers. std::vector headers(kHeaderCount); + headers[kMagicNumberOffset] = kCacheMagicNumber; headers[kCodeSizeOffset] = entry->code_size; headers[kCacheSizeOffset] = cache_size; headers[kCodeHashOffset] = entry->code_hash; headers[kCacheHashOffset] = cache_hash; - Debug("[compile cache] writing cache for %s in %s [%d %d %d %d]...", + Debug("[compile cache] writing cache for %s in %s [%d %d %d %d %d]...", entry->source_filename, entry->cache_filename, + headers[kMagicNumberOffset], headers[kCodeSizeOffset], headers[kCacheSizeOffset], headers[kCodeHashOffset], @@ -335,53 +358,63 @@ CompileCacheHandler::CompileCacheHandler(Environment* env) // Directory structure: // - Compile cache directory (from NODE_COMPILE_CACHE) -// - : hash of CachedDataVersionTag + NODE_VERESION -// - -// - -// - : a hash of filename + module type -// - -// - -bool CompileCacheHandler::InitializeDirectory(Environment* env, - const std::string& dir) { - compiler_cache_key_ = GetCacheVersionTag(); - std::string compiler_cache_key_string = Uint32ToHex(compiler_cache_key_); - std::vector paths = {dir, compiler_cache_key_string}; - std::string cache_dir = PathResolve(env, paths); - +// - $NODE_VERION-$ARCH-$CACHE_DATA_VERSION_TAG-$UID +// - $FILENAME_AND_MODULE_TYPE_HASH.cache: a hash of filename + module type +CompileCacheEnableResult CompileCacheHandler::Enable(Environment* env, + const std::string& dir) { + std::string cache_tag = GetCacheVersionTag(); + std::string absolute_cache_dir_base = PathResolve(env, {dir}); + std::filesystem::path cache_dir_with_tag = + std::filesystem::path(absolute_cache_dir_base) / cache_tag; + std::u8string cache_dir_with_tag_u8 = cache_dir_with_tag.u8string(); + std::string cache_dir_with_tag_str(cache_dir_with_tag_u8.begin(), + cache_dir_with_tag_u8.end()); + CompileCacheEnableResult result; Debug("[compile cache] resolved path %s + %s -> %s\n", dir, - compiler_cache_key_string, - cache_dir); + cache_tag, + cache_dir_with_tag_str); if (UNLIKELY(!env->permission()->is_granted( - env, permission::PermissionScope::kFileSystemWrite, cache_dir))) { - Debug("[compile cache] skipping cache because write permission for %s " - "is not granted\n", - cache_dir); - return false; + env, + permission::PermissionScope::kFileSystemWrite, + cache_dir_with_tag_str))) { + result.message = "Skipping compile cache because write permission for " + + cache_dir_with_tag_str + " is not granted"; + result.status = CompileCacheEnableStatus::kFailed; + return result; } if (UNLIKELY(!env->permission()->is_granted( - env, permission::PermissionScope::kFileSystemRead, cache_dir))) { - Debug("[compile cache] skipping cache because read permission for %s " - "is not granted\n", - cache_dir); - return false; + env, + permission::PermissionScope::kFileSystemRead, + cache_dir_with_tag_str))) { + result.message = "Skipping compile cache because read permission for " + + cache_dir_with_tag_str + " is not granted"; + result.status = CompileCacheEnableStatus::kFailed; + return result; } fs::FSReqWrapSync req_wrap; - int err = fs::MKDirpSync(nullptr, &(req_wrap.req), cache_dir, 0777, nullptr); + int err = fs::MKDirpSync( + nullptr, &(req_wrap.req), cache_dir_with_tag_str, 0777, nullptr); if (is_debug_) { Debug("[compile cache] creating cache directory %s...%s\n", - cache_dir, + cache_dir_with_tag_str, err < 0 ? uv_strerror(err) : "success"); } if (err != 0 && err != UV_EEXIST) { - return false; + result.message = + "Cannot create cache directory: " + std::string(uv_strerror(err)); + result.status = CompileCacheEnableStatus::kFailed; + return result; } - compile_cache_dir_ = std::filesystem::path(cache_dir); - return true; + compile_cache_dir_str_ = absolute_cache_dir_base; + result.cache_directory = absolute_cache_dir_base; + compile_cache_dir_ = cache_dir_with_tag; + result.status = CompileCacheEnableStatus::kEnabled; + return result; } } // namespace node diff --git a/src/compile_cache.h b/src/compile_cache.h index 018ccbdc114792..fa22c7f04a122a 100644 --- a/src/compile_cache.h +++ b/src/compile_cache.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "v8.h" @@ -34,10 +35,27 @@ struct CompileCacheEntry { v8::ScriptCompiler::CachedData* CopyCache() const; }; +#define COMPILE_CACHE_STATUS(V) \ + V(kFailed) /* Failed to enable the cache */ \ + V(kEnabled) /* Was not enabled before, and now enabled. */ \ + V(kAlreadyEnabled) /* Was already enabled. */ + +enum class CompileCacheEnableStatus : uint8_t { +#define V(status) status, + COMPILE_CACHE_STATUS(V) +#undef V +}; + +struct CompileCacheEnableResult { + CompileCacheEnableStatus status; + std::string cache_directory; + std::string message; // Set in case of failure. +}; + class CompileCacheHandler { public: explicit CompileCacheHandler(Environment* env); - bool InitializeDirectory(Environment* env, const std::string& dir); + CompileCacheEnableResult Enable(Environment* env, const std::string& dir); void Persist(); @@ -50,6 +68,7 @@ class CompileCacheHandler { void MaybeSave(CompileCacheEntry* entry, v8::Local mod, bool rejected); + std::string_view cache_dir() { return compile_cache_dir_str_; } private: void ReadCacheFile(CompileCacheEntry* entry); @@ -62,19 +81,18 @@ class CompileCacheHandler { template inline void Debug(const char* format, Args&&... args) const; - static constexpr size_t kCodeSizeOffset = 0; - static constexpr size_t kCacheSizeOffset = 1; - static constexpr size_t kCodeHashOffset = 2; - static constexpr size_t kCacheHashOffset = 3; - static constexpr size_t kHeaderCount = 4; + static constexpr size_t kMagicNumberOffset = 0; + static constexpr size_t kCodeSizeOffset = 1; + static constexpr size_t kCacheSizeOffset = 2; + static constexpr size_t kCodeHashOffset = 3; + static constexpr size_t kCacheHashOffset = 4; + static constexpr size_t kHeaderCount = 5; v8::Isolate* isolate_ = nullptr; bool is_debug_ = false; + std::string compile_cache_dir_str_; std::filesystem::path compile_cache_dir_; - // The compile cache is stored in a directory whose name is the hex string of - // compiler_cache_key_. - uint32_t compiler_cache_key_ = 0; std::unordered_map> compiler_cache_store_; }; diff --git a/src/env.cc b/src/env.cc index 096f0752e23f29..f2c0105fa28291 100644 --- a/src/env.cc +++ b/src/env.cc @@ -1129,15 +1129,36 @@ void Environment::InitializeCompileCache() { dir_from_env.empty()) { return; } - auto handler = std::make_unique(this); - if (handler->InitializeDirectory(this, dir_from_env)) { - compile_cache_handler_ = std::move(handler); - AtExit( - [](void* env) { - static_cast(env)->compile_cache_handler()->Persist(); - }, - this); + EnableCompileCache(dir_from_env); +} + +CompileCacheEnableResult Environment::EnableCompileCache( + const std::string& cache_dir) { + CompileCacheEnableResult result; + + if (!compile_cache_handler_) { + std::unique_ptr handler = + std::make_unique(this); + result = handler->Enable(this, cache_dir); + if (result.status == CompileCacheEnableStatus::kEnabled) { + compile_cache_handler_ = std::move(handler); + AtExit( + [](void* env) { + static_cast(env)->compile_cache_handler()->Persist(); + }, + this); + } + if (!result.message.empty()) { + Debug(this, + DebugCategory::COMPILE_CACHE, + "[compile cache] %s\n", + result.message); + } + } else { + result.status = CompileCacheEnableStatus::kAlreadyEnabled; + result.cache_directory = compile_cache_handler_->cache_dir(); } + return result; } void Environment::ExitEnv(StopFlags::Flags flags) { diff --git a/src/env.h b/src/env.h index 85bce29fcba427..ee119eb3556b6d 100644 --- a/src/env.h +++ b/src/env.h @@ -1028,6 +1028,9 @@ class Environment final : public MemoryRetainer { inline CompileCacheHandler* compile_cache_handler(); inline bool use_compile_cache() const; void InitializeCompileCache(); + // Enable built-in compile cache if it has not yet been enabled. + // The cache will be persisted to disk on exit. + CompileCacheEnableResult EnableCompileCache(const std::string& cache_dir); void RunAndClearNativeImmediates(bool only_refed = false); void RunAndClearInterrupts(); diff --git a/test/parallel/test-compile-cache-permission-disallowed.js b/test/parallel/test-compile-cache-permission-disallowed.js index 2f00da958217fc..dbbb38fb99f240 100644 --- a/test/parallel/test-compile-cache-permission-disallowed.js +++ b/test/parallel/test-compile-cache-permission-disallowed.js @@ -39,7 +39,7 @@ function testDisallowed(dummyDir, cacheDirInPermission, cacheDirInEnv) { }, { stderr(output) { - assert.match(output, /skipping cache because write permission for .* is not granted/); + assert.match(output, /Skipping compile cache because write permission for .* is not granted/); return true; } }); @@ -63,7 +63,7 @@ function testDisallowed(dummyDir, cacheDirInPermission, cacheDirInEnv) { }, { stderr(output) { - assert.match(output, /skipping cache because write permission for .* is not granted/); + assert.match(output, /Skipping compile cache because write permission for .* is not granted/); return true; } }); @@ -86,7 +86,7 @@ function testDisallowed(dummyDir, cacheDirInPermission, cacheDirInEnv) { }, { stderr(output) { - assert.match(output, /skipping cache because read permission for .* is not granted/); + assert.match(output, /Skipping compile cache because read permission for .* is not granted/); return true; } });