From de184b9974313a4ed14c4b10ab58c352428fe890 Mon Sep 17 00:00:00 2001 From: "zuoshu.zs" Date: Mon, 6 Dec 2021 11:27:43 +0800 Subject: [PATCH] add support for FUSE on Polar File System --- Readme-FUSE-CN.md | 155 +++++++++ Readme-FUSE.md | 155 +++++++++ deploy_scripts/mount_pfs_fuse.sh | 121 +++++++ deploy_scripts/start_pfsd.sh | 13 +- deploy_scripts/stop_pfsd.sh | 6 +- deploy_scripts/umount_pfs_fuse.sh | 87 +++++ install.sh | 15 +- src/CMakeLists.txt | 1 + src/pfs_fuse/CMakeLists.txt | 55 ++++ src/pfs_fuse/pfs_fuse.cc | 511 ++++++++++++++++++++++++++++++ src/pfs_fuse/pfs_fuse_main.cc | 36 +++ uninstall.sh | 9 +- 12 files changed, 1154 insertions(+), 10 deletions(-) create mode 100644 Readme-FUSE-CN.md create mode 100644 Readme-FUSE.md create mode 100755 deploy_scripts/mount_pfs_fuse.sh create mode 100755 deploy_scripts/umount_pfs_fuse.sh create mode 100644 src/pfs_fuse/CMakeLists.txt create mode 100644 src/pfs_fuse/pfs_fuse.cc create mode 100644 src/pfs_fuse/pfs_fuse_main.cc diff --git a/Readme-FUSE-CN.md b/Readme-FUSE-CN.md new file mode 100644 index 0000000..d1d38e1 --- /dev/null +++ b/Readme-FUSE-CN.md @@ -0,0 +1,155 @@ +# FUSE on pfs + +用户态文件系统（FUSE）是Unix和类似Unix的计算机操作系统的一个软件接口，允许非特权用户在不编辑内核代码的情况下创建自己的文件系统。这是通过在用户空间运行文件系统代码来实现的，而FUSE模块只提供了一个通往实际内核接口的桥梁。 + +为了让用户像其他内核内置文件系统一样使用PFS的内核接口，我们实现了一个FUSE on PFS的处理程序，它被链接到LibFUSE库。这个程序定义了内核接口与 PFS 操作的请求-响应映射，即它指定了PFS如何响应读/写/统计请求。同时，这个程序也被用来挂载 PFS，在挂载PFS的时候，处理程序被注册到内核中。如果用户现在对PFS发出读/写/统计请求，内核会将这些IO请求转发给处理程序，然后将处理程序的响应发回给用户。 + +# FUSE主要模块 + +##### FUSE内核模块（内核状态） + +FUSE内核模块实现了VFS接口（它实现了fuse文件驱动、fuse设备驱动的注册，并提供了超级块、inode等的维护）。它接收来自VFS的请求并将其传递给LibFUSE，然后LibFUSE将请求传递给PFS处理程序； + +##### LibFUSE模块（用户态） + +LibFUSE实现了文件系统的主要框架、PFS操作的封装、挂载管理以及通过/dev/fuse设备与内核模块的通信； + +##### 用户程序模块（用户态） + +用户程序在用户空间实现由LibFUSE库封装的PFS操作。 + +# 接口 + +FUSE接口在`fusepfs_operations`中定义，主要分为以下几类。 + +1. FUSE环境构建：init, destroy + +2. 文件操作：create, mknod, open, rename, truncate, ftruncate + +3. 目录操作：mkdir, opendir, readdir, rmdir + +4. 链接：symlink, readlink, unlink + +5. 文件属性：statfs, access, getattr, fgetattr + +6. 扩展属性：getxattr, setxattr, listxattr, removexattr + +7. 读写：read, write, read_buf, write_buf, fallocate + +8. 同步I/O：fsync, fsyncdir + +9. 多路复用：poll + +10. 释放：release, releasedir + +11. 其他：ioctl, lock, bmap + +# 使用FUSE on PFS + +#### 1. 安装PFS依赖 + +安装步骤详见文档[Readme-CN.md](./Readme-CN.md)中的【安装依赖】 + +#### 2. 加载FUSE模块 + +##### I. 下载FUSE资源包并解压 + +```bash +tar -zxvf fuse.tar.gz +``` + +推荐FUSE版本：2.9.2 + +##### II. 安装FUSE（3.2版本或以上需要安装Meson或Ninj） + +```bash +./configure && sudo make install +``` + +##### III. 检查是否加载成功 + +```bash +# 检查FUSE模块是否加载成功 +lsmod | grep fuse +# 如果尚未加载成功，你可以通过以下命令来挂载FUSE +modprobe fuse +# 查看版本信息 +fusermount --version +``` + +#### 3. 编译与安装 + +依赖项准备好后，进入代码根目录，执行脚本进行编译： + +```bash +./autobuild.sh && sudo ./install.sh +``` + +#### 4. 使用FUSE + +##### I. 挂载FUSE on PFS + +挂载前需要对/etc/fuse.conf进行配置，在文件中添加一行`user_allow_other`即可 + +```bash +/usr/local/polarstore/pfsd/bin/mount_pfs_fuse.sh [-p diskname] [-c rw/ro] mount_dir +# 示例 +/usr/local/polarstore/pfsd/bin/mount_pfs_fuse.sh -p nvme1n1 -c rw ./fuse_mntdir +``` + +`diskname`代表块设备名称。可以通过命令`lsblk`列出所有可用块设备信息； + +`rw/ro`代表所启动实例为启动读写实例或者只读实例； + +`mount_dir`指fuse挂载目录。 + +说明：挂载FUSE on PFS首先会启动pfsdaemon后台进程，然后启动pfs-fuse进程并挂载PFS到制定目录。 + +##### II. 通过FUSE访问PFS + +在后台启动FUSE实例后，现在你可以 `cd` 进入挂载目录，像平常使用内核内置的文件系统一样操作PFS。所有操作的结果将被送入PFS挂载的磁盘。示例如下： + +```bash +# 进入挂载目录 +$cd path/to/fuse_mount_dir +# 创建文件 写文件 +$echo "hello pfs fuse">test_file.txt +# 打印文件内容 +$cat test_file.txt +hello pfs fuse +``` + +##### III. 结束使用 + +1. 解挂FUSE on PFS + +你可以通过单独指定一个挂载路径来解挂特定FUSE实例，路径需要以绝对路径的形式指定； +也可以选择`all`参数来解挂所有已挂载的PFS FUSE实例。 + +```bash +/usr/local/polarstore/pfsd/bin/umount_pfs_fuse.sh [mount_dir/all] +# 示例 +/usr/local/polarstore/pfsd/bin/umount_pfs_fuse.sh /fuse_mntdir +/usr/local/polarstore/pfsd/bin/umount_pfs_fuse.sh all +``` + +2. 停止pfsdaemon后台进程 + +支持两种方式： +1. 通过盘名以停止指定pfsdaemon +2. 以停止所有正在运行的pfsdaemon + +```bash +sudo /usr/local/polarstore/pfsd/bin/stop_pfsd.sh [diskname] +sudo /usr/local/polarstore/pfsd/bin/stop_pfsd.sh + +example: +sudo /usr/local/polarstore/pfsd/bin/stop_pfsd.sh nvme1n1 +sudo /usr/local/polarstore/pfsd/bin/stop_pfsd.sh +``` + +##### IV. 使用卸载脚本uninstall.sh进行卸载： + +```bash +sudo ./uninstall.sh +``` \ No newline at end of file diff --git a/Readme-FUSE.md b/Readme-FUSE.md new file mode 100644 index 0000000..df899b9 --- /dev/null +++ b/Readme-FUSE.md @@ -0,0 +1,155 @@ +# FUSE on PFS + +Filesystem in USErspace (FUSE) is a software interface for Unix and Unix-like computer operating systems that lets non-privileged users create their own file systems without editing kernel code. This is achieved by running file system code in user space while the FUSE module provides only a bridge to the actual kernel interfaces. + +To allow users to use PFS with kernel interfaces like any other kernel built-in filesystems, we implement a FUSE on PFS handler program, which is linked to the supplied LibFUSE library. This program defines the request-respond mapping of kernal interfaces to pfs operations, which means it specify how PFS is to respond to read/write/stat requests. The program is also used to mount PFS. At the time PFS is mounted, the handler is registered with the kernel. If a user now issues read/write/stat requests for PFS, the kernel forwards these IO-requests to the handler and then sends the handler's response back to the user. + +# Main FUSE Modules + +#### FUSE kernel module (kernel state) + +The FUSE kernel module implements the VFS interface (which implements the registration of the fuse file driver, the fuse device driver, and provides maintenance of super blocks, inode, etc.) It receives requests from VFS and passes them to LibFUSE, and then LibFUSE passes requests to PFS handler program; + +#### LibFUSE module (user state) + +LibFUSE implements the main framework of the file system, the encapsulation of PFS operations, mount management and communication with the kernel module via /dev/fuse device; + +#### User program module (user state) + +User programs implement PFS operations encapsulated by the LibFUSE library in user space. + +# Interfaces + +FUSE interfaces are defined in`fusepfs_operations`, mainly divided into the following categories : + +1. FUSE environment building: init, destroy + +2. file operations: create, mknod, open, rename, truncate, ftruncate + +3. directory operations: mkdir, opendir, readdir, rmdir + +4. link: symlink, readlink, unlink + +5. file attribute: statfs, access, getattr, fgetattr + +6. extended attribute: getxattr, setxattr, listxattr, removexattr + +7. R/W: read, write, read_buf, write_buf, fallocate + +8. Sync I/O: fsync, fsyncdir + +9. multiplexing: poll + +10. release: release, releasedir + +11. other: ioctl, lock, bmap + + +# Use FUSE on PFS + +#### 1. Install PFS Dependencies + +Refer to 【Install Dependencies】part in document [Readme.md](./Readme.md) for installation steps + +#### 2. Load FUSE Module + +##### I. Download FUSE resource package and decompress + +``` bash +tar -zxvf fuse.tar.gz +``` +Recommended FUSE version: 2.9.2 + +##### II. Install FUSE (fuse 3.2 or above needs Meson or Ninj) + +```bash +./configure && sudo make install +``` + +##### III. Check + +````bash +# check if FUSE is mounted successfully +lsmod | grep fuse +# If not, you can use `modprobe fuse` to mount FUSE. +modprobe fuse +# Look up version information +fusermount --version +```` + +#### 3. Complie and Install + +After the dependencies are installed, go to the root directory of PFS source code and run the script to compile and install PFS. + +```bash +./autobuild.sh && sudo ./install.sh +``` + +#### 4. Usage + +##### I. mount FUSE on PFS + +Before mounting, you need to configure /etc/fuse.conf: +add `user_allow_other` to the file + +```bash +/usr/local/polarstore/pfsd/bin/mount_pfs_fuse.sh [-p diskname] [-c rw/ro] mount_dir +# example +/usr/local/polarstore/pfsd/bin/mount_pfs_fuse.sh -p nvme1n1 -c rw ./fuse_mntdir +``` + +`diskname` block device. you can get the information of all your available block devices with shell command `lsblk`; + +`rw/ro` startup a read&write or read-only instance; + +`mount_dir` fuse mount directory. + +p.s. Mounting FUSE on PFS will first start pfsdaemon in the background, then start pfs-fuse process, pfs-fuse will mount PFS to the specified directory. + +##### II. Visit PFS by FUSE + +After starting a pfsdfuse instance in background, now you can `cd` into the mount directory to operate PFS like a kernel built-in file system as usual. The results of all operations will be sent into the disk mounted via PFS. Here is an example : + +```bash +# enter mount directory +$cd path/to/fuse_mount_dir +# create file and write +$echo "hello pfs fuse">test_file.txt +# show new file +$cat test_file.txt +hello pfs fuse +``` + +##### III. Stop using FUSE on PFS + +1. Umount FUSE on pfs + + +```bash +/usr/local/polarstore/pfsd/bin/umount_pfs_fuse.sh [mount_dir/all] +# example +/usr/local/polarstore/pfsd/bin/umount_pfs_fuse.sh /fuse_mntdir +/usr/local/polarstore/pfsd/bin/umount_pfs_fuse.sh all +``` + +your can appoint a `mount_dir` to umount the selected instance, `mount dir` should be pointed as absolute path; +your can also choose `all` to umount all mounted pfsdfuse instance. + +2. Stop pfsdaemon + +You can stop single pfsdaemon by pointing a diskname of the pfsdaemon, or you can kill all pfsdaemons by running stop_pfsd.sh without any parameter. + +```bash +sudo /usr/local/polarstore/pfsd/bin/stop_pfsd.sh [diskname] +sudo /usr/local/polarstore/pfsd/bin/stop_pfsd.sh + +example: +sudo /usr/local/polarstore/pfsd/bin/stop_pfsd.sh nvme1n1 +sudo /usr/local/polarstore/pfsd/bin/stop_pfsd.sh +``` + +##### IV. Run the uninstall.sh script to uninstall pfsdaemon + +``` +sudo ./uninstall.sh +``` \ No newline at end of file diff --git a/deploy_scripts/mount_pfs_fuse.sh b/deploy_scripts/mount_pfs_fuse.sh new file mode 100755 index 0000000..1dfdfa3 --- /dev/null +++ b/deploy_scripts/mount_pfs_fuse.sh @@ -0,0 +1,121 @@ +#! /bin/sh + +# Copyright (c) 2017-2021, Alibaba Group Holding Limited +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BASE_DIR=$(cd "$(dirname "$0")"; pwd) +FUSE_BIN=${BASE_DIR}/pfs-fuse + +usage(){ + echo "[mount_pfs_fuse.sh] usage: " + echo " /usr/local/polarstore/pfsd/bin/mount_pfs_fuse.sh [-p diskname] [-c rw/ro] mount_dir" + echo "example:" + echo " /usr/local/polarstore/pfsd/bin/mount_pfs_fuse.sh -p nvme1n1 -c rw fuse/mount/dir" + echo "logger location:" + echo " /var/log/pfs-[disk].log" +} + +if [ $# -eq 0 ]; then + usage + exit +else +# parameters check + if [ $# -ne 5 ]; then + usage + exit + fi + + # PFS options + if [ "$1" = '-p' ] && [ "$3" = '-c' ]; then + DISK_NAME=$2 + PFS_FLAGS=$4 + elif [ "$1" = '-c' ] && [ "$3" = '-p' ]; then + PFS_FLAGS=$2 + DISK_NAME=$4 + else + echo "Invalid arguments" + usage + exit + fi + MNT_DIR=$5 + + if [ "$PFS_FLAGS" = 'rw' ] || [ "$PFS_FLAGS" = 'RW' ];then + is_rw=1 + elif [ "$PFS_FLAGS" = 'ro' ] || [ "$PFS_FLAGS" = 'RO' ];then + is_rw=0 + else + echo "-c (read/write flag) should be followed by 'rw'(read and write)/'ro'(read only)" + usage + exit 1 + fi + + # check if pfdameon exist + pfsd_exist="ps -ef | grep pfsdaemon | grep -w '\-p $DISK_NAME' | wc -l" + exist_ret0=$(eval $pfsd_exist) + + if [ $exist_ret0 -eq 0 ]; then + mkdir -p /var/run/pfs + mkdir -p /var/run/pfsd + mkdir -p /dev/shm/pfsd + + chmod 777 /var/run/pfs + chmod 777 /var/run/pfsd + chmod 777 /dev/shm/pfsd + + CONF_FILE=${BASE_DIR}/../conf + + ulimit -c unlimited + ${BASE_DIR}/../bin/pfsdaemon -p ${DISK_NAME} -c ${CONF_FILE}/pfsd_logger.conf + + sleep 1 + + # check if start success + exist_ret0=$(eval $pfsd_exist) + if [ $exist_ret0 -eq 0 ]; then + echo "pfsdaemon $DISK_NAME start failed" + usage + exit 1 + fi + + echo "pfsdaemon $DISK_NAME start success" + fi + + # parameters check + echo "[mount_pfs_fuse.sh] pfs fuse mount..." + mntdir_exist="ps -ef | grep 'pfs-fuse' | grep -w '$MNT_DIR' | wc -l" + exist_ret1=$(eval $mntdir_exist) + # if multi-write instance on ont disk + diskname_rw_exist="ps -ef | grep 'pfs-fuse' | grep -w '$DISK_NAME' | grep -w 'rw'| wc -l" + exist_ret2=$(eval $diskname_rw_exist) + if [ $exist_ret1 -ge 1 ]; then + echo "[mount_pfs_fuse.sh] mount error: path $MNT_DIR is already mounted" + fi + if [ $is_rw -eq 1 ] && [ $exist_ret2 -ge 1 ]; then + echo "[mount_pfs_fuse.sh] mount error: disk $DISK_NAME is already mounted with a rw instance! a disk can only be mounted with one rw instance, you can mount a ro instance or choose another disk." + exit 1 + fi + + # pfs fuse mount + sudo $FUSE_BIN -s -o allow_other -o direct_io -o auto_unmount --pbdname=$DISK_NAME --flags=$PFS_FLAGS $MNT_DIR + sleep 1 + # check if pfs fuse mount success + mount_exist="ps -ef | grep 'pfs-fuse' | grep -w '$MNT_DIR' | grep -w '$DISK_NAME' | grep -w '$PFS_FLAGS' | wc -l" + exist_ret=$(eval $mount_exist) + if [ $exist_ret -eq 0 ]; then + echo "[mount_pfs_fuse.sh] pfs fuse mount failed!" + usage + exit 1 + else + echo "[mount_pfs_fuse.sh] pfs fuse mount success!" + fi +fi diff --git a/deploy_scripts/start_pfsd.sh b/deploy_scripts/start_pfsd.sh index ef7c511..84d327a 100755 --- a/deploy_scripts/start_pfsd.sh +++ b/deploy_scripts/start_pfsd.sh @@ -13,6 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +usage(){ + echo "[start_pfsd.sh] usage: " + echo " sudo /usr/local/polarstore/pfsd/bin/start_pfsd.sh -p DISKNAME [other options]" + echo "example:" + echo " sudo /usr/local/polarstore/pfsd/bin/start_pfsd.sh -p nvme1n1" + echo "logger location:" + echo " /var/log/pfsd-DISKNAME/pfsd.log" +} + mkdir -p /var/run/pfs mkdir -p /var/run/pfsd mkdir -p /dev/shm/pfsd @@ -30,6 +39,7 @@ exist_command="ps -ef | grep pfsdaemon |grep -w '$pbdname' | wc -l" exist=$(eval $exist_command) if [ $exist -ge 1 ]; then echo "$pbdname already exist" + usage exit 1 fi @@ -39,10 +49,11 @@ ${BASE_DIR}/../bin/pfsdaemon $* -c ${CONF_FILE}/pfsd_logger.conf sleep 1 -# check if start success +# check if start success exist=$(eval $exist_command) if [ $exist -eq 0 ]; then echo "pfsdaemon $pfsdname start failed" + usage exit 1 fi diff --git a/deploy_scripts/stop_pfsd.sh b/deploy_scripts/stop_pfsd.sh index 1ac5b05..755b45c 100755 --- a/deploy_scripts/stop_pfsd.sh +++ b/deploy_scripts/stop_pfsd.sh @@ -27,7 +27,7 @@ kill_single_pfsd() { kill -2 $pid sleep 1 - # check if stop success, if not, use kill -9 + # check if stop success, if not, use kill -9 exist=$(eval $exist_command) if [ $exist -eq 0 ]; then echo "$pfsdname stop success" @@ -46,8 +46,8 @@ kill_single_pfsd() { } kill_all_pfsd() { - pkill -2 pfsdaemon - sleep 1 + pkill -2 pfsdaemon + sleep 1 cnt=`ps -ef | grep pfsdaemon | grep -v grep | wc -l` if [ $cnt -eq 0 ]; then echo "pkill -2 all pfsdaemon success" diff --git a/deploy_scripts/umount_pfs_fuse.sh b/deploy_scripts/umount_pfs_fuse.sh new file mode 100755 index 0000000..5ee8c31 --- /dev/null +++ b/deploy_scripts/umount_pfs_fuse.sh @@ -0,0 +1,87 @@ +#! /bin/sh + +# Copyright (c) 2017-2021, Alibaba Group Holding Limited +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +usage(){ + echo "[umount_pfs_fuse.sh] usage: " + echo " /usr/local/polarstore/pfsd/bin/umount_pfs_fuse.sh [absolute_mount_dir/all]" + echo "example:" + echo " /usr/local/polarstore/pfsd/bin/umount_pfs_fuse.sh /fuse/mount/dir" + echo " /usr/local/polarstore/pfsd/bin/umount_pfs_fuse.sh all" + echo "logger location:" + echo " /var/log/pfs-[disk].log" +} + +kill_single_fuse() { + echo "[umount_pfs_fuse.sh] umount pfs fuse $MNTDIR..." + MNTDIR=$1 + + sudo umount -l $MNT_DIR + sleep 1 + + exist_command="ps -ef | grep 'pfs-fuse' | grep -w '$MNTDIR' | wc -l" + exist=$(eval $exist_command) + if [ $exist -eq 0 ]; then + echo "[umount_pfs_fuse.sh] pfs fuse $MNT_DIR umount success!" + exit 0 + fi + + ps -ef |grep 'pfs-fuse' | grep -w $MNTDIR | awk '{print $2}' | xargs kill -9 + sleep 1 + + exist=$(eval $exist_command) + if [ $exist -eq 0 ]; then + echo "[umount_pfs_fuse.sh] pfs fuse $MNT_DIR umount success!" + exit 0 + else + echo "[umount_pfs_fuse.sh] pfs fuse $MNT_DIR umount failed!" + usage + fi +} + +kill_all_fuse() { + echo "[umount_pfs_fuse.sh] umount all pfs fuses..." + + ps -ef |grep 'pfs-fuse' | grep 'pbdname' | awk '{print $2}'| xargs kill -9 + sleep 1 + + exist_command="ps -ef | grep 'pfs-fuse' | grep 'pbdname' | wc -l" + exist=$(eval $exist_command) + + if [ $exist -eq 0 ]; then + echo "[umount_pfs_fuse.sh] umount all pfs fuses success!" + exit 0 + else + echo "[umount_pfs_fuse.sh] pfs fuse umount failed!" + usage + fi +} + +# umount pfs fuse +if [ $# -ne 1 ]; then + usage + exit +else + # umount fuse + echo "[umount_pfs_fuse.sh] pfs fuse umount..." + if [ "$1" == 'all' ]; then + kill_all_fuse + elif [ "$1" = '--help' ] || [ "$1" = '-h' ]; then + usage + exit + else + MNT_DIR=$1 + kill_single_fuse $MNT_DIR + fi +fi diff --git a/install.sh b/install.sh index ad7ddc9..4b752ec 100755 --- a/install.sh +++ b/install.sh @@ -28,16 +28,19 @@ if [ -f "${INSTALL_BASE_DIR}/pfsd/include/pfsd_sdk.h" ] || \ [ -f "${INSTALL_BASE_DIR}/pfsd/lib/libpfsd.a" ] || \ [ -f "${INSTALL_BASE_DIR}/pfsd/lib/libpfsd_test.so" ] || \ [ -f "${INSTALL_BASE_DIR}/pfsd/bin/pfsdaemon" ] || \ +[ -f "${INSTALL_BASE_DIR}/pfsd/bin/pfs-fuse" ] || \ [ -f "${INSTALL_BASE_DIR}/pfsd/bin/pfsd_shm_tool" ] || \ [ -f "${INSTALL_BASE_DIR}/pfsd/conf/pfsd_logger.conf" ] || \ [ -f "${INSTALL_BASE_DIR}/pfsd/bin/start_pfsd.sh" ] || \ [ -f "${INSTALL_BASE_DIR}/pfsd/bin/stop_pfsd.sh" ] || \ +[ -f "${INSTALL_BASE_DIR}/pfsd/bin/mount_pfs_fuse.sh" ] || \ +[ -f "${INSTALL_BASE_DIR}/pfsd/bin/umount_pfs_fuse.sh" ] || \ [ -f "${INSTALL_BASE_DIR}/pfsd/bin/clean_pfsd.sh" ] || \ [ -f "/etc/init.d/pfsd_env" ] || \ [ -f "/etc/polarfs.conf" ] || \ [ -f "/usr/local/bin/pfs" ] || \ [ -f "/usr/local/bin/pfsadm" ];then - echo "pfsd has installed, install failed" + echo "pfsd/fuse has installed, install failed" exit 1 fi @@ -45,10 +48,13 @@ if [ ! -f "src/pfs_sdk/pfsd_sdk.h" ] || \ [ ! -f "lib/libpfsd.a" ] || \ [ ! -f "lib/libpfsd_test.so" ] || \ [ ! -f "bin/pfsdaemon" ] || \ +[ ! -f "bin/pfs-fuse" ] || \ [ ! -f "bin/pfsd_shm_tool" ] || \ [ ! -f "conf/pfsd_logger.conf" ] || \ [ ! -f "deploy_scripts/start_pfsd.sh" ] || \ [ ! -f "deploy_scripts/stop_pfsd.sh" ] || \ +[ ! -f "deploy_scripts/mount_pfs_fuse.sh" ] || \ +[ ! -f "deploy_scripts/umount_pfs_fuse.sh" ] || \ [ ! -f "deploy_scripts/clean_pfsd.sh" ] || \ [ ! -f "src/pfsd/pfsd.init" ] || \ [ ! -f "etc/polarfs.conf" ] || \ @@ -59,7 +65,7 @@ if [ ! -f "src/pfs_sdk/pfsd_sdk.h" ] || \ fi if [[ $EUID -ne 0 ]];then - echo "pfsd install script must be run as root" + echo "pfsd/fuse install script must be run as root" exit 1 fi @@ -68,10 +74,13 @@ install -m 0644 src/pfs_sdk/pfsd_sdk.h ${INSTALL_BASE_DIR}/pfsd/include/pfsd_s install -m 0755 lib/libpfsd.a ${INSTALL_BASE_DIR}/pfsd/lib/libpfsd.a install -m 0755 lib/libpfsd_test.so ${INSTALL_BASE_DIR}/pfsd/lib/libpfsd_test.so install -m 0755 bin/pfsdaemon ${INSTALL_BASE_DIR}/pfsd/bin/pfsdaemon +install -m 0755 bin/pfs-fuse ${INSTALL_BASE_DIR}/pfsd/bin/pfs-fuse install -m 0755 bin/pfsd_shm_tool ${INSTALL_BASE_DIR}/pfsd/bin/pfsd_shm_tool install -m 0644 conf/pfsd_logger.conf ${INSTALL_BASE_DIR}/pfsd/conf/pfsd_logger.conf install -m 0755 deploy_scripts/start_pfsd.sh ${INSTALL_BASE_DIR}/pfsd/bin/start_pfsd.sh install -m 0755 deploy_scripts/stop_pfsd.sh ${INSTALL_BASE_DIR}/pfsd/bin/stop_pfsd.sh +install -m 0755 deploy_scripts/mount_pfs_fuse.sh ${INSTALL_BASE_DIR}/pfsd/bin/mount_pfs_fuse.sh +install -m 0755 deploy_scripts/umount_pfs_fuse.sh ${INSTALL_BASE_DIR}/pfsd/bin/umount_pfs_fuse.sh install -m 0755 deploy_scripts/clean_pfsd.sh ${INSTALL_BASE_DIR}/pfsd/bin/clean_pfsd.sh install -m 0755 src/pfsd/pfsd.init /etc/init.d/pfsd_env install -m 0644 etc/polarfs.conf /etc/polarfs.conf @@ -89,4 +98,4 @@ chmod 777 /var/run/pfs touch /var/run/pfsd/.pfsd chkconfig --add pfsd_env -echo "install pfsd success!" \ No newline at end of file +echo "install pfsd success!" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1b44d42..2a9c39d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,3 +15,4 @@ add_subdirectory(pfs_core) add_subdirectory(pfs_sdk) add_subdirectory(pfsd) add_subdirectory(pfs_tools) +add_subdirectory(pfs_fuse) diff --git a/src/pfs_fuse/CMakeLists.txt b/src/pfs_fuse/CMakeLists.txt new file mode 100644 index 0000000..d67a9ca --- /dev/null +++ b/src/pfs_fuse/CMakeLists.txt @@ -0,0 +1,55 @@ +# Copyright (c) 2017-2021, Alibaba Group Holding Limited +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include_directories( + ${PROJECT_SOURCE_DIR}/src/pfs_fuse/ + ${PROJECT_SOURCE_DIR}/src/pfs_sdk/ + ${PROJECT_SOURCE_DIR}/src/trace/include/ +) + +link_directories( + ${LIBRARY_OUTPUT_PATH} +) + +SET(SRC_LIST_FUSE pfs_fuse_main.cc + pfs_fuse.cc + ) + +add_executable(pfs-fuse + ${SRC_LIST_FUSE} +) + +add_definitions(-D_FILE_OFFSET_BITS=64) + +set(CXXFLAGS + -Werror=no-literal-suffix + -g + -O0 +) + +if(${CMAKE_HOST_SYSTEM_PROCESSOR} STREQUAL "x86_64") + set(CXXFLAGS ${CXXFLAGS} -march=native) +endif() + +target_link_libraries(pfs-fuse + -Wl,--start-group + -Wl,--no-as-needed + pfsd + pthread + fuse + -Wl,--end-group +) + +add_dependencies(pfs-fuse + pfsd +) diff --git a/src/pfs_fuse/pfs_fuse.cc b/src/pfs_fuse/pfs_fuse.cc new file mode 100644 index 0000000..9853575 --- /dev/null +++ b/src/pfs_fuse/pfs_fuse.cc @@ -0,0 +1,511 @@ +/* + * Copyright (c) 2017-2021, Alibaba Group Holding Limited + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define FUSE_USE_VERSION 29 +#define FUSE_CONF ((struct pfs_fuse_conf_t *) fuse_get_context()->private_data) + +#include +#include +#include +#include +#include +#include +#include +#include "pfsd_sdk.h" + +struct pfs_fuse_conf_t { + char *pbdname; + char *flags; + int conf; +}; + +static struct fuse_opt pfs_fuse_opt[] = { + { "--pbdname=%s", offsetof(struct pfs_fuse_conf_t, pbdname), 0 }, + { "--flags=%s", offsetof(struct pfs_fuse_conf_t, flags), 0 }, + { "--conf", offsetof(struct pfs_fuse_conf_t, conf), 1 }, + FUSE_OPT_END +}; + +/* FUSE handlers */ +static void * +fusepfs_init(struct fuse_conn_info *conn) +{ + int rv; + if (strcmp(FUSE_CONF->flags, "rw")==0 || strcmp(FUSE_CONF->flags, "RW")==0) + rv = pfsd_mount("disk", FUSE_CONF->pbdname, 0, PFS_RDWR); + else if (strcmp(FUSE_CONF->flags, "ro")==0 || strcmp(FUSE_CONF->flags, "RO")==0) + rv = pfsd_mount("disk", FUSE_CONF->pbdname, 0, PFS_RD); + else + rv = EINVAL; + if (rv != 0) + exit(-1); + + return FUSE_CONF; +} + +static void +fusepfs_destroy(void *private_data) +{ + (void)pfsd_umount(FUSE_CONF->pbdname); +} + +static int +fusepfs_create(const char *path, mode_t mode, struct fuse_file_info *fi) +{ + char pbdpath[PFS_MAX_PATHLEN]; + int fd; + int rv; + + rv = snprintf(pbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, path); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + fd = pfsd_creat(pbdpath, mode); + if (fd < 0) + return -errno; + + fi->fh = fd; + return 0; +} + +static int +fusepfs_open(const char *path, struct fuse_file_info *fi) +{ + char pbdpath[PFS_MAX_PATHLEN]; + int fd; + int rv; + + rv = snprintf(pbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, path); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + fd = pfsd_open(pbdpath, fi->flags, S_IRWXU | S_IRWXG | S_IRWXO); + if (fd < 0) + return -errno; + + fi->fh = fd; + return 0; +} + +static int +fusepfs_rename(const char *path, const char *newpath) +{ + char pbdpath[PFS_MAX_PATHLEN]; + char newpbdpath[PFS_MAX_PATHLEN]; + int rv; + + rv = snprintf(pbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, path); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + rv = snprintf(newpbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, newpath); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + rv = pfsd_rename(pbdpath, newpbdpath); + if (rv < 0) + return -errno; + + return 0; +} + +static int +fusepfs_truncate(const char *path, off_t newsize) +{ + char pbdpath[PFS_MAX_PATHLEN]; + int rv; + + rv = snprintf(pbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, path); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + rv = pfsd_truncate(pbdpath, newsize); + if (rv < 0) + return -errno; + + return 0; +} + +static int +fusepfs_ftruncate(const char *path, off_t off, struct fuse_file_info *fi) +{ + int rv; + + rv = pfsd_ftruncate(fi->fh, off); + if (rv < 0) + return -errno; + + return 0; +} + +static int +fusepfs_mkdir(const char *path, mode_t mode) +{ + char pbdpath[PFS_MAX_PATHLEN]; + int rv; + + rv = snprintf(pbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, path); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + rv = pfsd_mkdir(pbdpath, mode); + if (rv < 0) + return -errno; + + return 0; +} + +static int +fusepfs_opendir(const char *path, struct fuse_file_info *fi) +{ + char pbdpath[PFS_MAX_PATHLEN]; + DIR *dp; + int rv; + + rv = snprintf(pbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, path); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + dp = pfsd_opendir(pbdpath); + if (dp == NULL) + return -errno; + + fi->fh = (intptr_t)dp; + return 0; +} + +static int +fusepfs_readdir(const char *path, void *buffer, fuse_fill_dir_t filler, off_t off, struct fuse_file_info *fi) +{ + DIR *dp; + struct dirent *de; + + dp = (DIR *) (uintptr_t) fi->fh; + de = pfsd_readdir(dp); + if (de == NULL) + return -errno; + + do { + if (filler(buffer, de->d_name, NULL, 0) != 0) { + return -ENOMEM; + } + } while ((de = pfsd_readdir(dp)) != NULL); + + return 0; +} + +static int +fusepfs_rmdir(const char *path) +{ + char pbdpath[PFS_MAX_PATHLEN]; + int rv; + + rv = snprintf(pbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, path); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + rv = pfsd_rmdir(pbdpath); + if (rv < 0) + return -errno; + + return 0; +} + +static int +fusepfs_readlink(const char *path, char *link, size_t size) +{ + char pbdpath[PFS_MAX_PATHLEN]; + int rv; + + rv = snprintf(pbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, path); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + rv = pfsd_readlink(pbdpath, link, size - 1); + if (rv < 0) + return -errno; + + link[rv] = '\0'; + return 0; +} + +static int +fusepfs_unlink(const char *path) +{ + char pbdpath[PFS_MAX_PATHLEN]; + int rv; + + rv = snprintf(pbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, path); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + rv = pfsd_unlink(pbdpath); + if (rv < 0) + return -errno; + + return 0; +} + +static int +fusepfs_access(const char *path, int mask) +{ + char pbdpath[PFS_MAX_PATHLEN]; + int rv; + + rv = snprintf(pbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, path); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + rv = pfsd_access(pbdpath, mask); + if (rv < 0) + return -errno; + + return rv; +} + +static int +fusepfs_getattr(const char *path, struct stat *st) +{ + char pbdpath[PFS_MAX_PATHLEN]; + int rv; + + rv = snprintf(pbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, path); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + rv = pfsd_stat(pbdpath, st); + if (rv < 0) + return -errno; + + return 0; +} + +static int +fusepfs_chmod(const char *path, mode_t mode) +{ + char pbdpath[PFS_MAX_PATHLEN]; + int rv; + + rv = snprintf(pbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, path); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + rv = pfsd_chmod(pbdpath, mode);; + if (rv < 0) + return -errno; + + return 0; +} + +static int +fusepfs_chown(const char *path, uid_t uid, gid_t gid) +{ + char pbdpath[PFS_MAX_PATHLEN]; + int rv; + + rv = snprintf(pbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, path); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + rv = pfsd_chown(pbdpath, uid, gid);; + if (rv < 0) + return -errno; + + return 0; +} + +static int +fusepfs_read(const char *path, char *buffer, size_t size, off_t off, struct fuse_file_info *fi) +{ + ssize_t rlen; + + rlen = pfsd_pread(fi->fh, buffer, size, off); + if (rlen < 0) + return -errno; + + return rlen; +} + +static int +fusepfs_write(const char *path, const char *buffer, size_t size, off_t off, struct fuse_file_info *fi) +{ + ssize_t wlen; + + wlen = pfsd_pwrite(fi->fh, buffer, size, off); + if (wlen < 0) + return -errno; + + return wlen; +} + +static int +fusepfs_fallocate(const char *path, int mode, off_t off, off_t length, struct fuse_file_info *fi) +{ + char pbdpath[PFS_MAX_PATHLEN]; + int file_mode = S_IRWXU | S_IRWXG | S_IRWXO; + int fd, rv; + + rv = snprintf(pbdpath, PFS_MAX_PATHLEN, "/%s%s", FUSE_CONF->pbdname, path); + if (rv >= PFS_MAX_PATHLEN) + return -errno; + + if (fi == NULL) { + fd = pfsd_open(pbdpath, O_RDWR, file_mode); + if (fd < 0) + return -errno; + } else { + fd = fi->fh; + if (fd < 0) + return -EINVAL; + } + + rv = pfsd_fallocate(fd, mode, off, length); + + if (fi == NULL) + (void)pfsd_close(fd); + + if (rv < 0) + return -errno; + + return rv; +} + +static int +fusepfs_fsync(const char *path, int datasync, struct fuse_file_info *fi) +{ + int rv; + + rv = pfsd_fsync(fi->fh); + if (rv < 0) + return -errno; + + return 0; +} + +static int +fusepfs_release(const char *path, struct fuse_file_info *fi) +{ + int rv; + + rv = pfsd_close(fi->fh); + if (rv < 0) + return -errno; + + return 0; +} + +static int +fusepfs_releasedir(const char *path, struct fuse_file_info *fi) +{ + int rv; + + rv = pfsd_closedir((DIR *) (uintptr_t) fi->fh); + if (rv < 0) + return -errno; + + return 0; +} + +static int +fusepfs_utimens(const char *path, const struct timespec tv[2]) +{ + return 0; +} + +static int +fusepfs_utime(const char *, struct utimbuf *) +{ + return 0; +} + +static struct fuse_operations fusepfs_operations = { + + .getattr = fusepfs_getattr, // Get file attr specified by filepath, like lstat() + .readlink = fusepfs_readlink, // Read the target of a symbolic link + .getdir = NULL, + .mknod = NULL, // Create non-dir, non-symlink nodes, rarely needed + .mkdir = fusepfs_mkdir, // Create a dir + .unlink = fusepfs_unlink, // Remove a file + .rmdir = fusepfs_rmdir, // Remove a file + .symlink = NULL, // Create a symbolic link + .rename = fusepfs_rename, // Rename a file, dir, or other obj + .link = NULL, // Create a hard link + .chmod = fusepfs_chmod, // Change the permission bits of a file + .chown = fusepfs_chown, // Change the owner and group of a file + .truncate = fusepfs_truncate, + .utime = fusepfs_utime, + .open = fusepfs_open, + .read = fusepfs_read, // Read data from an open file + .write = fusepfs_write, // Write data to an open file + .statfs = NULL, // Get fs statistics + .flush = NULL, // Possibly flush cached data (just flush the data, waiting for async finish) + .release = fusepfs_release, // Release an open file, called when no reference to an open file, see close() + .fsync = fusepfs_fsync, // Sync file contents to disk + .setxattr = NULL, + .getxattr = NULL, + .listxattr = NULL, + .removexattr = NULL, + .opendir = fusepfs_opendir, // Open a dir, return fd + .readdir = fusepfs_readdir, // Read a dir, iterate all entries of some dir, return info in buffer + .releasedir = fusepfs_releasedir, // Release dir + .fsyncdir = NULL, // Sync dir contents + .init = fusepfs_init, // Initialize fs, one-time setup like private_data. return private_data + .destroy = fusepfs_destroy, // Clean up fs, paired with init + .access = fusepfs_access, // Check file access permissions + .create = fusepfs_create, // Create and open a file + .ftruncate = fusepfs_ftruncate, + .fgetattr = NULL, + .lock = NULL, // Perform POSIX file locking operation + .utimens = fusepfs_utimens, // Change the access and modification times of a file with nanosecond resolution + .bmap = NULL, // Map block index within file to block index within device / + .flag_nullpath_ok = 0, + .flag_nopath = 0, + .flag_utime_omit_ok = 0, + .flag_reserved = 0, + .ioctl = NULL, // Ioctl + .poll = NULL, // Poll for IO readiness events + .write_buf = NULL, // Write contents of buffer to an open file + .read_buf = NULL, // Store data from an open file in a buffer + .flock = NULL, // Perform BSD file locking operation + .fallocate = fusepfs_fallocate, // Allocates space for an open file +}; + +static void +print_fuse_conf(struct pfs_fuse_conf_t *fuse_conf) +{ + fprintf(stdout, "****** CONFIG ******\n"); + fprintf(stdout, "\tpbdname:\t%s\n", fuse_conf->pbdname); + fprintf(stdout, "\tflags: \t%s\n", fuse_conf->flags); + fprintf(stdout, "********************\n"); +} + +int +pfs_fuse_main(int argc, char *argv[]) +{ + struct fuse_args args = FUSE_ARGS_INIT(argc, argv); + struct pfs_fuse_conf_t fuse_conf; + + memset(&fuse_conf, 0, sizeof(fuse_conf)); + if (fuse_opt_parse(&args, &fuse_conf, pfs_fuse_opt, NULL) != 0){ + return 1; + } + + fprintf(stdout, "starting fuse[%d] %s\n", getpid(), fuse_conf.pbdname); + + print_fuse_conf(&fuse_conf); + + /* Turn over control to fuse */ + return fuse_main(args.argc, args.argv, &fusepfs_operations, &fuse_conf); +} diff --git a/src/pfs_fuse/pfs_fuse_main.cc b/src/pfs_fuse/pfs_fuse_main.cc new file mode 100644 index 0000000..3ad3e56 --- /dev/null +++ b/src/pfs_fuse/pfs_fuse_main.cc @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2017-2021, Alibaba Group Holding Limited + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +extern int pfs_fuse_main(int argc, char *argv[]); + +int main(int argc, char *argv[]) +{ + int rv; + + rv = pfs_fuse_main(argc, argv); + if(rv < 0){ + fprintf(stderr, "[pfsd fuse] pfs_fuse_main error, err=%d\n", rv); + return -1; + } + + return 0; +} diff --git a/uninstall.sh b/uninstall.sh index dba25fc..39be3e1 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -16,10 +16,10 @@ INSTALL_BASE_DIR="/usr/local/polarstore" #uninstall check -exist_command="ps -ef | grep pfsdaemon | grep -v grep | wc -l" +exist_command="ps -ef | grep -E 'pfsdaemon|pfs-fuse' | grep -v grep | wc -l" exist=$(eval $exist_command) if [ $exist -ge 1 ]; then - echo "pfsd is running, before uninstall pfsd, please stop pfsd" + echo "pfsd or fuse is running, before uninstall pfsd, please stop pfsd and fuse" exit 1 fi @@ -33,10 +33,13 @@ rm ${INSTALL_BASE_DIR}/pfsd/include/pfsd_sdk.h rm ${INSTALL_BASE_DIR}/pfsd/lib/libpfsd.a rm ${INSTALL_BASE_DIR}/pfsd/lib/libpfsd_test.so rm ${INSTALL_BASE_DIR}/pfsd/bin/pfsdaemon +rm ${INSTALL_BASE_DIR}/pfsd/bin/pfs-fuse rm ${INSTALL_BASE_DIR}/pfsd/bin/pfsd_shm_tool rm ${INSTALL_BASE_DIR}/pfsd/conf/pfsd_logger.conf rm ${INSTALL_BASE_DIR}/pfsd/bin/start_pfsd.sh rm ${INSTALL_BASE_DIR}/pfsd/bin/stop_pfsd.sh +rm ${INSTALL_BASE_DIR}/pfsd/bin/mount_pfs_fuse.sh +rm ${INSTALL_BASE_DIR}/pfsd/bin/umount_pfs_fuse.sh rm ${INSTALL_BASE_DIR}/pfsd/bin/clean_pfsd.sh rm /etc/init.d/pfsd_env rm /etc/polarfs.conf @@ -44,4 +47,4 @@ rm /etc/polarfs.conf rm /usr/local/bin/pfs rm /usr/local/bin/pfsadm -echo "uninstall pfsd success!" \ No newline at end of file +echo "uninstall pfsd success!"