前言
Curve 是云原生计算基金会 (CNCF) Sandbox 项目,是网易主导自研和开源的高性能、易运维、云原生的分布式存储系统。
FUSE 文件系统开发实践 - "hello world"
实现 fuse_lowlevel_ops 结构体对应的接口
struct fuse_lowlevel_ops {void (*init) (void *userdata, struct fuse_conn_info *conn);void (*destroy) (void *userdata);void (*lookup) (fuse_req_t req, fuse_ino_t parent, const char *name);void (*forget) (fuse_req_t req, fuse_ino_t ino, uint64_t nlookup);void (*getattr) (fuse_req_t req, fuse_ino_t ino,struct fuse_file_info *fi);void (*setattr) (fuse_req_t req, fuse_ino_t ino, struct stat *attr,int to_set, struct fuse_file_info *fi);void (*readlink) (fuse_req_t req, fuse_ino_t ino);void (*mknod) (fuse_req_t req, fuse_ino_t parent, const char *name,mode_t mode, dev_t rdev);void (*mkdir) (fuse_req_t req, fuse_ino_t parent, const char *name,mode_t mode);// .... 还有其他很多接口}
创建 fuse session
创建挂载
工作
#define FUSE_USE_VERSION 34#include <fuse_lowlevel.h>#include <stdio.h>#include <stdlib.h>#include <string.h>#include <errno.h>#include <fcntl.h>#include <unistd.h>#include <assert.h>static const char *hello_str = "Hello World!\n";static const char *hello_name = "hello";// fuse_lowlevel_op接口函数实现static int hello_stat(fuse_ino_t ino, struct stat *stbuf){stbuf->st_ino = ino;switch (ino) {case 1:stbuf->st_mode = S_IFDIR | 0755;stbuf->st_nlink = 2;break;case 2:stbuf->st_mode = S_IFREG | 0444;stbuf->st_nlink = 1;stbuf->st_size = strlen(hello_str);break;default:return -1;}return 0;}// fuse_lowlevel_op接口函数实现static void hello_ll_getattr(fuse_req_t req, fuse_ino_t ino,struct fuse_file_info *fi){struct stat stbuf;(void) fi;memset(&stbuf, 0, sizeof(stbuf));if (hello_stat(ino, &stbuf) == -1)fuse_reply_err(req, ENOENT);elsefuse_reply_attr(req, &stbuf, 1.0);}// fuse_lowlevel_op接口函数实现static void hello_ll_lookup(fuse_req_t req, fuse_ino_t parent, const char *name){struct fuse_entry_param e;if (parent != 1 || strcmp(name, hello_name) != 0)fuse_reply_err(req, ENOENT);else {memset(&e, 0, sizeof(e));e.ino = 2;e.attr_timeout = 1.0;e.entry_timeout = 1.0;hello_stat(e.ino, &e.attr);fuse_reply_entry(req, &e);}}struct dirbuf {char *p;size_t size;};static void dirbuf_add(fuse_req_t req, struct dirbuf *b, const char *name,fuse_ino_t ino){struct stat stbuf;size_t oldsize = b->size;b->size += fuse_add_direntry(req, NULL, 0, name, NULL, 0);b->p = (char *) realloc(b->p, b->size);memset(&stbuf, 0, sizeof(stbuf));stbuf.st_ino = ino;fuse_add_direntry(req, b->p + oldsize, b->size - oldsize, name, &stbuf,b->size);}#define min(x, y) ((x) < (y) ? (x) : (y))static int reply_buf_limited(fuse_req_t req, const char *buf, size_t bufsize,off_t off, size_t maxsize){if (off < bufsize)return fuse_reply_buf(req, buf + off,min(bufsize - off, maxsize));elsereturn fuse_reply_buf(req, NULL, 0);}// fuse_lowlevel_op接口函数实现static void hello_ll_readdir(fuse_req_t req, fuse_ino_t ino, size_t size,off_t off, struct fuse_file_info *fi){(void) fi;if (ino != 1)fuse_reply_err(req, ENOTDIR);else {// 当发生readdir调用调用时,返回hello_name文件名struct dirbuf b;memset(&b, 0, sizeof(b));dirbuf_add(req, &b, ".", 1);dirbuf_add(req, &b, "..", 1);dirbuf_add(req, &b, hello_name, 2);reply_buf_limited(req, b.p, b.size, off, size);free(b.p);}}// fuse_lowlevel_op接口函数实现static void hello_ll_open(fuse_req_t req, fuse_ino_t ino,struct fuse_file_info *fi){if (ino != 2)fuse_reply_err(req, EISDIR);else if ((fi->flags & O_ACCMODE) != O_RDONLY)fuse_reply_err(req, EACCES);elsefuse_reply_open(req, fi);}// fuse_lowlevel_op接口函数实现static void hello_ll_read(fuse_req_t req, fuse_ino_t ino, size_t size,off_t off, struct fuse_file_info *fi){(void) fi;assert(ino == 2);// 当发生read调用时,返回hello_str中的字符串reply_buf_limited(req, hello_str, strlen(hello_str), off, size);}// fuse_lowlevel_op接口实现static const struct fuse_lowlevel_ops hello_ll_oper = {.lookup = hello_ll_lookup,.getattr = hello_ll_getattr,.readdir = hello_ll_readdir,.open = hello_ll_open,.read = hello_ll_read,};int main(int argc, char *argv[]){struct fuse_args args = FUSE_ARGS_INIT(argc, argv);struct fuse_session *se;struct fuse_cmdline_opts opts;struct fuse_loop_config config;int ret = -1;// 参数解析if (fuse_parse_cmdline(&args, &opts) != 0)return 1;if (opts.show_help) {printf("usage: %s [options] <mountpoint>\n\n", argv[0]);fuse_cmdline_help();fuse_lowlevel_help();ret = 0;goto err_out1;} else if (opts.show_version) {printf("FUSE library version %s\n", fuse_pkgversion());fuse_lowlevel_version();ret = 0;goto err_out1;}if(opts.mountpoint == NULL) {printf("usage: %s [options] <mountpoint>\n", argv[0]);printf(" %s --help\n", argv[0]);ret = 1;goto err_out1;}// 创建fuse sessionse = fuse_session_new(&args, &hello_ll_oper,sizeof(hello_ll_oper), NULL);if (se == NULL)goto err_out1;if (fuse_set_signal_handlers(se) != 0)goto err_out2;// 挂载if (fuse_session_mount(se, opts.mountpoint) != 0)goto err_out3;fuse_daemonize(opts.foreground);// 开始工作/* Block until ctrl+c or fusermount -u */if (opts.singlethread)ret = fuse_session_loop(se);else {config.clone_fd = opts.clone_fd;config.max_idle_threads = opts.max_idle_threads;ret = fuse_session_loop_mt(se, &config);}fuse_session_unmount(se);err_out3:fuse_remove_signal_handlers(se);err_out2:fuse_session_destroy(se);err_out1:free(opts.mountpoint);fuse_opt_free_args(&args);return ret ? 1 : 0;}
编译
挂载以及使用
// 创建挂载目录root@hostname:~$ mkdir mountpoint// 挂载root@hostname:~$ ./hello mountpoint// 演示root@hostname:~$ cd mountpointroot@hostname:~/mountpoint$root@hostname:~/mountpoint$ lshelloroot@hostname:~/mountpoint$ cat helloHello World!// 卸载 - 卸载失败,因为挂载点被占用root@hostname:~/mountpoint$ sudo umount ~/mountpointumount: home/root/mountpoint: target is busy(In some cases useful info about processes thatuse the device is found by lsof(8) or fuser(1).)// 退出挂载点目录root@hostname:~/mountpoint$ cd// 卸载 - 成功root@hostname:~$ sudo umount ~/mountpoint
性能迷雾
This patch series make maximum read/write request size tunable in FUSE.Currently, it is limited to FUSE_MAX_PAGES_PER_REQ which is equalto 32 pages. It is required to change it in order to improve thethroughput since optimized value depends on various factors suchas type and version of local filesystems used and HW specs, etc.In addition, recently FUSE is widely used as a gateway to connectcloud storage services and distributed filesystems. Larger datamight be stored in them over networking via FUSE and the overheadmight affect the throughput.It seems there were many requests to increase FUSE_MAX_PAGES_PER_REQto improve the throughput,
问题解决
root@pubt1-ceph73:/mnt/test1# echo 1024 > sys/module/fuse/parameters/fuse_max_pages_per_req
root@hostname:~/github_linux/linux/fs/fuse$ git show 5da784cce4308commit 5da784cce4308ae10a79e3c8c41b13fb9568e4e0Author: Constantine Shulyupin <const@MakeLinux.com>Date: Thu Sep 6 15:37:06 2018 +0300fuse: add max_pages to init_outReplace FUSE_MAX_PAGES_PER_REQ with the configurable parameter max_pages toimprove performance.Old RFC with detailed description of the problem and many fixes by MitsuoHayasaka (mitsuo.hayasaka.hu@hitachi.com):- https://lkml.org/lkml/2012/7/5/136We've encountered performance degradation and fixed it on a big and complexvirtual environment.
上述性能调优只是 Curve 文件存储众多性能调优手段的其中之一,后续我们将会继续分享更多的性能优化方案,欢迎感兴趣的小伙伴关注共建社区。
参考文献
AI 场景中的 IO 行为浅析


关于 Curve
Curve 亦可作为云存储中间件使用 S3 兼容的对象存储作为数据存储引擎,为公有云用户提供高性价比的共享文件存储。
GitHub:https://github.com/opencurve/curve 官网:https://opencurve.io/ 用户论坛:https://ask.opencurve.io/ 微信群:搜索群助手微信号 OpenCurve_bot
文章转载自OpenCurve,如果涉嫌侵权,请发送邮件至:contact@modb.pro进行举报,并提供相关证据,一经查实,墨天轮将立刻删除相关内容。




