lib/9pfs: Return ENOTTY on terminal ioctl calls

Some applications (e.g. python) will use `ioctl()` calls to check if an
interactive interpretor should be started or if a file should be read.

`9pfs` should respond to all terminal-related calls with an `ENOTTY`
error, so applications know that they are not running in an interactive
environment.

In order to detect the ioctl request type, add a new macro,
`IOCTL_CMD_ISTYPE(cmd, type)`, which will check the corresponding bytes
from the request number (an enhanced version of the Linux `_IOC_TYPE`).

Signed-off-by: Stefan Jumarea <[email protected]>
Reviewed-by: Sergiu Moga <[email protected]>
Approved-by: Simon Kuenzer <[email protected]>
GitHub-Closes: unikraft#1098

lib/9pfs/9pfs_vnops.c

@@ -1015,7 +1015,6 @@ static int uk_9pfs_symlink(struct vnode *dvp, const char *op, const char *np)
 static int uk_9pfs_ioctl(struct vnode *dvp, struct vfscore_file *fp,
 			 unsigned long com, void *data)
 {
-	switch (com) {
 	/**
 	 * HACK: In binary compatibility mode, Ruby tries to set O_ASYNC,
 	 * which Unikraft does not yet support. If the `ioctl` call returns
@@ -1028,12 +1027,15 @@ static int uk_9pfs_ioctl(struct vnode *dvp, struct vfscore_file *fp,
 	 * Setting `ioctl` to a nullop will not work, since it is used by
 	 * interpreted languages (e.g. python3) to check if it should start
 	 * the interpretor or just read a file.
+	 *
+	 * For every `ioctl` request related to a terminal, return ENOTTY.
 	 */
-	case FIONBIO:
+	if (com == FIONBIO)
 		return 0;
-	default:
-		return ENOTSUP;
-	}
+	if (IOCTL_CMD_ISTYPE(com, IOCTL_CMD_TYPE_TTY))
+		return ENOTTY;
+
+	return ENOTSUP;
 }
 
 #define uk_9pfs_seek		((vnop_seek_t)vfscore_vop_nullop)

lib/vfscore/include/vfscore/vnode.h

@@ -43,6 +43,15 @@
 #include <vfscore/uio.h>
 #include <vfscore/dentry.h>
 
+#define IOCTL_CMD_TYPE_SHIFT		(8)
+#define IOCTL_CMD_TYPE_MASK		(0xFF << IOCTL_CMD_TYPE_SHIFT)
+#define IOCTL_CMD_TYPE_TTY		('T')
+
+#define IOCTL_CMD_ISTYPE(cmd, type)			\
+		((cmd & (IOCTL_CMD_TYPE_MASK)) ==	\
+		 (((type) << IOCTL_CMD_TYPE_SHIFT) &	\
+		  IOCTL_CMD_TYPE_MASK))
+
 struct vfsops;
 struct vnops;
 struct vnode;