Vine: Set File Mode

taskvine/src/bindings/python3/ndcctools/taskvine/file.py

@@ -45,6 +45,15 @@ def type(self):
         if self._file:
             return cvine.vine_file_type(self._file)
 
+    ##
+    # Set the Unix mode permission bits for the remote file.
+    #
+    # @param self       A file object.
+    # @param mode       Unix mode bits.
+    def set_mode(self, mode):
+        if self._file:
+            return cvine.vine_file_set_mode(self._file, mode)
+
     ##
     # Return the contents of a file object as a string.
     # Typically used to return the contents of an output buffer.

taskvine/src/manager/taskvine.h

@@ -889,6 +889,16 @@ be delete at the manager's site after it is not needed by the workflow (@ref vin
 struct vine_file *vine_declare_starch(
 		struct vine_manager *m, struct vine_file *f, vine_cache_level_t cache, vine_file_flags_t flags);
 
+/** Set the Unix mode permission bits of a declared file.
+Sets (or overrides) the Unix mode permissions of any file object,
+as the application will see it.  This applies to any file type,
+but is particularly useful for controlling buffers, urls, and mini-tasks
+that do not inherently contain mode bits.
+@param f A file object of any kind.
+@param mode The Unix mode bits to be applied to the file.
+*/
+void vine_file_set_mode( struct vine_file *f, int mode );
+
 /** Fetch the contents of a file.
 The contents of the given file will be loaded from disk or pulled back from the cluster
 and loaded into manager memory.  This is particularly useful for temporary files and mini-tasks

taskvine/src/manager/vine_file.c

@@ -85,6 +85,7 @@ struct vine_file *vine_file_create(const char *source, const char *cached_name,
 	f->source_worker = 0;
 	f->type = type;
 	f->size = size;
+	f->mode = 0;
 	f->mini_task = mini_task;
 	f->recovery_task = 0;
 	f->state = VINE_FILE_STATE_CREATED; /* Assume state created until told otherwise */
@@ -371,4 +372,11 @@ const char *vine_file_source(struct vine_file *f)
 	return f->source;
 }
 
+void vine_file_set_mode(struct vine_file *f, int mode)
+{
+	/* The mode must contain, at a minimum, owner-rw (0600) (so that we can delete it) */
+	/* And it should not contain anything beyond the standard 0777. */
+	f->mode = (mode | 0600) & 0777;
+}
+
 /* vim: set noexpandtab tabstop=8: */

taskvine/src/manager/vine_file.h

@@ -38,6 +38,7 @@ struct vine_file {
 	char *cached_name;  // Name of file in the worker's cache directory.
 	size_t size;        // Length of source data, if known.
 	time_t mtime;       // Modification time of source data, if known.
+	mode_t mode;        // Manual override for Unix mode bits sent to worker.  Zero if unset.
 	char *data;         // Raw data for an input or output buffer.
 	struct vine_task *mini_task; // Mini task used to generate the desired output file.
 	struct vine_task *recovery_task; // For temp files, a copy of the task that created it.

taskvine/src/manager/vine_manager_put.c

@@ -66,8 +66,7 @@ The transfer time is controlled by the size of the file.
 If the transfer takes too long, then cancel it.
 */
 
-static int vine_manager_put_file(
-		struct vine_manager *q, struct vine_worker_info *w, struct vine_task *t, const char *localname, const char *remotename, struct stat info, int64_t *total_bytes)
+static int vine_manager_put_file(struct vine_manager *q, struct vine_worker_info *w, struct vine_task *t, const char *localname, const char *remotename, struct stat info, int override_mode, int64_t *total_bytes)
 {
 	time_t stoptime;
 	timestamp_t effective_stoptime = 0;
@@ -76,6 +75,10 @@ static int vine_manager_put_file(
 	/* normalize the mode so as not to set up invalid permissions */
 	int mode = (info.st_mode | 0x600) & 0777;
 
+	/* If user provided override mode bits at the top level, use those instead. */
+	if (override_mode)
+		mode = override_mode;
+
 	int64_t length = info.st_size;
 
 	int fd = open(localname, O_RDONLY, 0);
@@ -113,23 +116,27 @@ static int vine_manager_put_file(
 /* Need prototype here to address mutually recursive code. */
 
 static vine_result_code_t vine_manager_put_file_or_dir(
-		struct vine_manager *q, struct vine_worker_info *w, struct vine_task *t, const char *name, const char *remotename, int64_t *total_bytes, int follow_links);
+		struct vine_manager *q, struct vine_worker_info *w, struct vine_task *t, const char *name, const char *remotename, int override_mode, int64_t *total_bytes, int follow_links);
 
 /*
 Send a directory and all of its contents using the new streaming protocol.
 Do this by sending a "dir" prefix, then all of the directory contents,
 and then an "end" marker.
 */
 
-static vine_result_code_t vine_manager_put_directory(
-		struct vine_manager *q, struct vine_worker_info *w, struct vine_task *t, const char *localname, const char *remotename, int64_t *total_bytes)
+static vine_result_code_t vine_manager_put_directory(struct vine_manager *q, struct vine_worker_info *w, struct vine_task *t, const char *localname, const char *remotename, int override_mode, int64_t *total_bytes)
 {
 	struct stat info;
 	if (stat(localname, &info) != 0) {
 		debug(D_NOTICE, "Cannot stat dir %s: %s", localname, strerror(errno));
 		return VINE_APP_FAILURE;
 	}
 
+	/* If user provided override mode bits at the top level, use those instead. */
+	int mode = info.st_mode;
+	if (override_mode)
+		mode = override_mode;
+
 	DIR *dir = opendir(localname);
 	if (!dir) {
 		debug(D_NOTICE, "Cannot open dir %s: %s", localname, strerror(errno));
@@ -141,7 +148,7 @@ static vine_result_code_t vine_manager_put_directory(
 	char remotename_encoded[VINE_LINE_MAX];
 	url_encode(remotename, remotename_encoded, sizeof(remotename_encoded));
 
-	vine_manager_send(q, w, "dir %s %0o %lld\n", remotename_encoded, info.st_mode, (long long)info.st_mtime);
+	vine_manager_send(q, w, "dir %s %0o %lld\n", remotename_encoded, mode, (long long)info.st_mtime);
 
 	struct dirent *d;
 	while ((d = readdir(dir))) {
@@ -150,7 +157,7 @@ static vine_result_code_t vine_manager_put_directory(
 
 		char *localpath = string_format("%s/%s", localname, d->d_name);
 
-		result = vine_manager_put_file_or_dir(q, w, t, localpath, d->d_name, total_bytes, 0);
+		result = vine_manager_put_file_or_dir(q, w, t, localpath, d->d_name, 0, total_bytes, 0);
 
 		free(localpath);
 
@@ -177,8 +184,7 @@ However, in recursive calls, follow_links is set to zero,
 and internal links are not followed, they are sent natively.
 */
 
-static vine_result_code_t vine_manager_put_file_or_dir(
-		struct vine_manager *q, struct vine_worker_info *w, struct vine_task *t, const char *localpath, const char *remotepath, int64_t *total_bytes, int follow_links)
+static vine_result_code_t vine_manager_put_file_or_dir(struct vine_manager *q, struct vine_worker_info *w, struct vine_task *t, const char *localpath, const char *remotepath, int override_mode, int64_t *total_bytes, int follow_links)
 {
 	struct stat info;
 	int result = VINE_SUCCESS;
@@ -191,11 +197,11 @@ static vine_result_code_t vine_manager_put_file_or_dir(
 
 	if (result >= 0) {
 		if (S_ISDIR(info.st_mode)) {
-			result = vine_manager_put_directory(q, w, t, localpath, remotepath, total_bytes);
+			result = vine_manager_put_directory(q, w, t, localpath, remotepath, override_mode, total_bytes);
 		} else if (S_ISLNK(info.st_mode)) {
 			result = vine_manager_put_symlink(q, w, t, localpath, remotepath, total_bytes);
 		} else if (S_ISREG(info.st_mode)) {
-			result = vine_manager_put_file(q, w, t, localpath, remotepath, info, total_bytes);
+			result = vine_manager_put_file(q, w, t, localpath, remotepath, info, override_mode, total_bytes);
 		} else {
 			debug(D_NOTICE, "skipping unusual file: %s", strerror(errno));
 		}
@@ -222,9 +228,12 @@ vine_result_code_t vine_manager_put_url_now(struct vine_manager *q, struct vine_
 		return VINE_SUCCESS;
 	}
 
-	/* XXX The API does not allow the user to choose the mode bits of the target file, so we make it permissive
-	 * here.*/
-	int mode = 0755;
+	/* A URL source does not naturally provide mode bits. */
+	/* If the user provided them manually via vine_file_set_mode, use that. */
+	/* Otherwise default to a permissive 0755. */
+	int mode = f->mode;
+	if (mode == 0)
+		mode = 0755;
 
 	char source_encoded[VINE_LINE_MAX];
 	char cached_name_encoded[VINE_LINE_MAX];
@@ -258,9 +267,12 @@ vine_result_code_t vine_manager_put_url(struct vine_manager *q, struct vine_work
 		return VINE_SUCCESS;
 	}
 
-	/* XXX The API does not allow the user to choose the mode bits of the target file, so we make it permissive
-	 * here.*/
-	int mode = 0755;
+	/* A URL source does not naturally provide mode bits. */
+	/* If the user provided them manually via vine_file_set_mode, use that. */
+	/* Otherwise default to a permissive 0755. */
+	int mode = f->mode;
+	if (mode == 0)
+		mode = 0755;
 
 	char source_encoded[VINE_LINE_MAX];
 	char cached_name_encoded[VINE_LINE_MAX];
@@ -283,8 +295,15 @@ vine_result_code_t vine_manager_put_url(struct vine_manager *q, struct vine_work
 
 vine_result_code_t vine_manager_put_buffer(struct vine_manager *q, struct vine_worker_info *w, struct vine_task *t, struct vine_file *f, int64_t *total_bytes)
 {
+	/* A buffer source does not naturally provide mode bits. */
+	/* If the user provided them manually via vine_file_set_mode, use that. */
+	/* Otherwise default to a permissive 0755. */
+	int mode = f->mode;
+	if (mode == 0)
+		mode = 0755;
+
 	time_t stoptime = time(0) + vine_manager_transfer_time(q, w, f->size);
-	vine_manager_send(q, w, "file %s %lld 0755 0\n", f->cached_name, (long long)f->size);
+	vine_manager_send(q, w, "file %s %lld 0%o 0\n", f->cached_name, (long long)f->size, (int)mode);
 	int64_t actual = link_putlstring(w->link, f->data, f->size, stoptime);
 	if (actual >= 0 && (size_t)actual == f->size) {
 		*total_bytes = actual;
@@ -311,7 +330,7 @@ static vine_result_code_t vine_manager_put_input_file(struct vine_manager *q, st
 	case VINE_FILE:
 		debug(D_VINE, "%s (%s) needs file %s as %s", w->hostname, w->addrport, f->source, m->remote_name);
 		vine_manager_send(q, w, "put %s %d %lld\n", f->cached_name, f->cache_level, (long long)f->size);
-		result = vine_manager_put_file_or_dir(q, w, t, f->source, f->cached_name, &total_bytes, 1);
+		result = vine_manager_put_file_or_dir(q, w, t, f->source, f->cached_name, f->mode, &total_bytes, 1);
 		break;
 
 	case VINE_BUFFER:
@@ -488,8 +507,14 @@ vine_result_code_t vine_manager_put_task(
 		return result;
 
 	if (target) {
-		vine_manager_send(q, w, "mini_task %s %s %d %lld %o\n", target->source, target->cached_name, target->cache_level, (long long)target->size, 0777);
+		/* If the user provide mode bits manually, use them here. */
+		int mode = target->mode;
+		if (mode == 0)
+			mode = 0755;
+		/* A mini-task is identified by the file it creates. */
+		vine_manager_send(q, w, "mini_task %s %s %d %lld 0%o\n", target->source, target->cached_name, target->cache_level, (long long)target->size, mode);
 	} else {
+		/* A regular task is simply identified by a task id. */
 		vine_manager_send(q, w, "task %lld\n", (long long)t->task_id);
 	}
 

taskvine/test/vine_python.py

@@ -234,18 +234,19 @@ def next_output_name():
 
     # Create an explicit minitask description to run curl
     minitask = vine.Task("curl https://www.nd.edu -o output")
-    intask = q.declare_minitask(minitask,"output")
+    infile = q.declare_minitask(minitask,"output")
+    infile.set_mode(0o600)
 
     # Now generate an input file from a shell command:
     t = vine.Task("wc -l infile")
-    t.add_input(intask, "infile")
+    t.add_input(infile, "infile")
     q.submit(t)
     t = q.wait(wait_time)
     report_task(t, "success", 0)
 
     # second time should have it cached (though we can't tell from here)
     t = vine.Task("wc -l infile")
-    t.add_input(intask, "infile")
+    t.add_input(infile, "infile")
     q.submit(t)
     t = q.wait(wait_time)
     report_task(t, "success", 0)