This issue was moved to a discussion.
You can continue the conversation there. Go to discussion →
macOS vm virtiofs concurrency issue #23061
Comments
github-actions
bot
added
macos
|
Has anyone reported this to apple? |
|
It has been reported and evidently fixed in 15? rust-lang/docker-rust#161 |
|
Okay after a bit of digging I actually found out what the docker devs did (Huge Thank You 🙏). It's basically this kernel patch: [PATCH 9/9] hardlinks: drop the cache of the existing entry.From 85589340b82880f5e52bee98f96ef11ae412dbb5 Mon Sep 17 00:00:00 2001
From: Docker <[email protected]>
Date: Sun, 11 Feb 2024 09:06:12 +0000
Subject: [PATCH 9/9] hardlinks: drop the cache of the existing entry.
Under virtualization.framework's virtiofs, the inode numbers are synthetic.
When the hardlink is created, the inode of the original file sometimes
changes too which means cached lookups will fail.
Work around by dropping the cache of the existing entry.
This doesn't affect grpcfuse as the host inode numbers are exposed in the VM.
Tested with a reproducer like this:
```
touch a
mkdir b
cat > main.c << EOT
int one(){
int ret;
struct stat st;
ret = stat("b/b", &st);
if (ret == 0) {
unlink("b/b");
}
if (!stat("b/b", &st)){
fprintf(stderr, "b/b exists\n");
return 0;
}
if (stat("a", &st)){
fprintf(stderr, "a does not exist\n");
return 0;
}
struct timespec begin, end;
clock_gettime(CLOCK_MONOTONIC_RAW, &begin);
ret = link("a", "b/b");
if(ret == -1){
perror("link");
if (stat("a", &st)){
fprintf(stderr, "a does not exist\n");
} else {
fprintf(stderr, "a exists\n");
}
if (stat("b", &st)){
fprintf(stderr, "b does not exist\n");
} else {
fprintf(stderr, "b exists\n");
}
if (stat("b/b", &st)){
fprintf(stderr, "b/b does not exist\n");
} else {
fprintf(stderr, "b/b exists\n");
}
int retries = 0;
do {
ret = link("a", "b/b");
if (ret == 0) {
break;
}
retries++;
} while (1);
clock_gettime(CLOCK_MONOTONIC_RAW, &end);
double time_spent = (end.tv_nsec - begin.tv_nsec) / 1000000000.0 + (end.tv_sec - begin.tv_sec);
fprintf(stderr, "link succeeded after %d retries in %f seconds\n", retries, time_spent);
return 0;
}
int fd = open("b/b", O_RDONLY);
if(fd == -1){
perror("open");
return 0;
}
sendfile(1, fd, NULL, 16777216);
close(fd);
ret = unlink("b/b");
if(ret == -1){
perror("unlink");
return 0;
}
return 1;
}
void main(){
for (int i = 0;; i++){
if (!one()){
fprintf(stdout, "\nFailed on iteration %d\n", i);
break;
};
fprintf(stdout, ".");
fflush(stdout);
}
}
EOT
Signed-off-by: Docker <[email protected]>
---
fs/fuse/dir.c | 20 +++++++++++++++++---
fs/fuse/readdir.c | 3 +++
2 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c
index d707e6987..05fd0fa27 100644
--- a/fs/fuse/dir.c
+++ b/fs/fuse/dir.c
@@ -258,6 +258,10 @@ static int fuse_dentry_revalidate(struct dentry *entry, unsigned int flags)
fuse_change_attributes(inode, &outarg.attr, NULL,
ATTR_TIMEOUT(&outarg),
attr_version);
+ if ((inode->i_nlink > 1) && (!S_ISDIR(inode->i_mode))){
+ /* This case happens a lot when using hardlinks */
+ outarg.entry_valid = 0;
+ }
fuse_change_entry_timeout(entry, &outarg);
} else if (inode) {
fi = get_fuse_inode(inode);
@@ -442,9 +446,12 @@ static struct dentry *fuse_lookup(struct inode *dir, struct dentry *entry,
goto out_err;
entry = newent ? newent : entry;
- if (outarg_valid)
+ if (outarg_valid) {
+ if (inode && (inode->i_nlink > 1) && (!S_ISDIR(inode->i_mode))){
+ outarg.entry_valid = 0;
+ }
fuse_change_entry_timeout(entry, &outarg);
- else
+ } else
fuse_invalidate_entry_cache(entry);
if (inode)
@@ -690,6 +697,9 @@ static int fuse_create_open(struct inode *dir, struct dentry *entry,
}
kfree(forget);
d_instantiate(entry, inode);
+ if ((inode->i_nlink > 1) && (!S_ISDIR(inode->i_mode))){
+ outentry.entry_valid = 0;
+ }
fuse_change_entry_timeout(entry, &outentry);
fuse_dir_changed(dir);
err = finish_open(file, entry, generic_file_open);
@@ -819,7 +829,9 @@ static int create_new_entry(struct fuse_mount *fm, struct fuse_args *args,
d = d_splice_alias(inode, entry);
if (IS_ERR(d))
return PTR_ERR(d);
-
+ if (args->opcode == FUSE_LINK){
+ outarg.entry_valid = 0;
+ }
if (d) {
fuse_change_entry_timeout(d, &outarg);
dput(d);
@@ -1101,6 +1113,7 @@ static int fuse_link(struct dentry *entry, struct inode *newdir,
struct fuse_link_in inarg;
struct inode *inode = d_inode(entry);
struct fuse_mount *fm = get_fuse_mount(inode);
+ struct fuse_entry_out not_valid = {0,0};
FUSE_ARGS(args);
memset(&inarg, 0, sizeof(inarg));
@@ -1111,6 +1124,7 @@ static int fuse_link(struct dentry *entry, struct inode *newdir,
args.in_args[0].value = &inarg;
args.in_args[1].size = newent->d_name.len + 1;
args.in_args[1].value = newent->d_name.name;
+ fuse_change_entry_timeout(entry, ¬_valid);
err = create_new_entry(fm, &args, newdir, newent, inode->i_mode);
if (!err)
fuse_update_ctime_in_cache(inode);
diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c
index 9e6d587b3..ab282724f 100644
--- a/fs/fuse/readdir.c
+++ b/fs/fuse/readdir.c
@@ -256,6 +256,9 @@ static int fuse_direntplus_link(struct file *file,
}
if (fc->readdirplus_auto)
set_bit(FUSE_I_INIT_RDPLUS, &get_fuse_inode(inode)->state);
+ if ((inode->i_nlink > 1) && (!S_ISDIR(inode->i_mode))) {
+ o->entry_valid = 0;
+ }
fuse_change_entry_timeout(dentry, o);
dput(dentry);
--
2.43.0 |
|
Quick update: I've built a local kernel with that patch applied (applies cleanly on 6.9.6) and it seems that it works as a temporary workaround. |
|
@cynecx do you have a link to that git commit or mailing list entry? i can't find it anywhere |
|
@kasperk81 You can find all the patches embedded in the docker vm itself under |
For what it's worth, I was not able to reproduce with these steps on my mac, but I also did not get a cross build as in your logs |
|
@cfergeau Oh yeah sorry about that. I was just being lazy and pasted another very related test-case's build output. But it's still really just a hello world with a single |
|
A friendly reminder that this issue had no activity for 30 days. |
|
I move this to a discussion as this is not a podman bug and we just ship the default fedora kernel so we have no way of patching the kernel. |
This issue was moved to a discussion.
You can continue the conversation there. Go to discussion →
Issue Description
It seems like there is a virtiofs issue somewhere (probably a bug in Virtualization.framework's virtiofs implementation) that causes file operations in the virtiofs mount to fail with "No such file or directory" errors.
Docker-for-mac had this same issue: docker/for-mac#7059. They've apparently fixed this somehow.
Steps to reproduce the issue
Steps to reproduce the issue
cargo new hello && cd hellopodman image pull rust:1-bookwormpodman run --rm -it -v $(pwd):/app -w /app rust:1-bookwormcargo clean && cargo buildDescribe the results you received
Describe the results you expected
The build commands should all run successfully. The build artifacts should be properly "visible".
podman info output
The text was updated successfully, but these errors were encountered: