commit 682ba1f45c0f601ef5c404fa7f2803222564f541 Author: indigo Date: Sun Dec 4 22:47:05 2022 +0800 Initial commit diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfe0770 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Auto detect text files and perform LF normalization +* text=auto diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7400abe --- /dev/null +++ b/.gitignore @@ -0,0 +1,35 @@ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app +/build/* +/install/* +/vendor/* diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..e423ff6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,4 @@ +{ + "python.pythonPath": "C:\\python27\\python.exe", + "python.formatting.provider": "yapf" +} \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..58d9b8d --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,4 @@ +project(pynfs) +cmake_minimum_required(VERSION 3.1.2) + +add_subdirectory(src) \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..cd38aab --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 indigo + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..60382ad --- /dev/null +++ b/README.md @@ -0,0 +1,39 @@ +PyNFS +===== +PyNFS is another python binding libnfs using pybind11. Now is only tested on windows platform + +Build and running +===== +### Build Requirement +* Visual Studio 2015 + +* GCC +* CMake + +### Windows +Please follow the next steps to build shared library on windows: + +```cmd +mkdir build +cd build +cmake -G "Visual Studio 15 2017" -DCMAKE_INSTALL_PREFIX ../install .. +cmake -build . -config Release -target install +``` + +### Linux +Please follow the next steps to build shared library on windows: +```cmd +mkdir build +cd build +cmake -G "Makefile" -DCMAKE_INSTALL_PREFIX ../install .. +cmake -build . -config Release -target install +``` + +### Example +Here is some example to demo the library usage +```python +from PyNFS import NFS, NFSSH +nfs= NFS('nfs://127.0.0.1/data/tmp/') +a = nfs.open('/test.txt', mode='w+') +a.write('Test String') +a.close() +``` \ No newline at end of file diff --git a/python/pynfs/__init__.py b/python/pynfs/__init__.py new file mode 100644 index 0000000..acbfc52 --- /dev/null +++ b/python/pynfs/__init__.py @@ -0,0 +1 @@ +from _pynfs import * \ No newline at end of file diff --git a/python/setup.py b/python/setup.py new file mode 100644 index 0000000..d901f87 --- /dev/null +++ b/python/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup, Extension + +setup(name='pynfs', + version='1.0', + author='indigo', + author_email='indigo.cgcg.com.tw', + description='Python binding for libnfs use pybind11') diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..66c997d --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,44 @@ +project(PyNFS) + +set(SRC_FILES + "pynfs.h" + "pynfs.cpp" + "NFS.h" + "NFS.cpp" + "NFSFH.h" + "NFSFH.cpp" + "wildcards.hpp" + "fileutils.hpp") + +add_definitions(-D_CRT_SECURE_NO_WARNINGS) + +find_package(Python2 COMPONENTS Interpreter Development REQUIRED) +message(STATUS "Project : ${PROJECT_NAME}") +message(STATUS "Python Site : ${Python2_SITELIB}") +message(STATUS "Python Include Dirs : ${Python2_INCLUDE_DIRS}") +message(STATUS "Python Library Dirs : ${Python2_LIBRARY_DIRS}") +message(STATUS "Python Library : ${Python2_LIBRARIES}") + +include_directories( + ${Python2_INCLUDE_DIRS} + "${CMAKE_SOURCE_DIR}/vendor/libnfs/include" + "${CMAKE_SOURCE_DIR}/vendor/pybind11/include" + "${CMAKE_SOURCE_DIR}/vendor/pystring/include" +) +link_directories( + ${Python2_LIBRARY_DIRS} + "${CMAKE_SOURCE_DIR}/vendor/libnfs/lib" + "${CMAKE_SOURCE_DIR}/vendor/pystring/lib" +) + +add_library(${PROJECT_NAME} SHARED ${SRC_FILES}) +add_custom_command(TARGET ${PROJECT_NAME} + POST_BUILD COMMAND "copy" ARGS "$(TargetDir)$(TargetFileName)" "${Python2_SITELIB}\\${PROJECT_NAME}" "/Y/B") +target_link_libraries(${PROJECT_NAME} nfs pystring ${Python2_LIBRARIES}) +set_target_properties(${PROJECT_NAME} PROPERTIES OUTPUT_NAME "_${PROJECT_NAME}") +set_target_properties(${PROJECT_NAME} PROPERTIES SUFFIX ".pyd") + +install(TARGETS ${PROJECT_NAME} + RUNTIME DESTINATION "${CMAKE_INSTALL_PREFIX}/${PROJECT_NAME}") +install(FILES "${CMAKE_SOURCE_DIR}/python/${PROJECT_NAME}/__init__.py" + DESTINATION ${CMAKE_INSTALL_PREFIX}/${PROJECT_NAME}) \ No newline at end of file diff --git a/src/NFS.cpp b/src/NFS.cpp new file mode 100644 index 0000000..1158325 --- /dev/null +++ b/src/NFS.cpp @@ -0,0 +1,761 @@ +#include "NFS.h" +#include "NFSFH.h" + +#include "fileutils.hpp" +#include "wildcards.hpp" + +namespace pynfs +{ + NFS::NFS() : _nfs(NULL), _url(NULL) + { +#ifdef WIN32 + WSADATA wsaData; + if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) + { + printf("Failed to start Winsock2\n"); + return; + } +#endif + _nfs = nfs_init_context(); + if (_nfs == NULL) + { + printf("failed to init context\n"); + return; + } + }; + + NFS::NFS(std::string path) : _nfs(NULL), _url(NULL) + { +#ifdef WIN32 + WSADATA wsaData; + if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) + { + printf("Failed to start Winsock2\n"); + return; + } +#endif + _nfs = nfs_init_context(); + if (_nfs == NULL) + { + printf("Failed to init context\n"); + return; + } + + if (openurl(path) != 0) + { + printf("Failed to openurl(\"%s\")", path.c_str()); + return; + } + } + + NFS::~NFS() + { + free_context(); + } + + int NFS::openurl(std::string path) + { + _url = nfs_parse_url_dir(_nfs, path.c_str()); + if (_url == NULL) + { + fprintf(stderr, "%s\n", nfs_get_error(_nfs)); + return 1; + } + + if (nfs_mount(_nfs, _url->server, _url->path) != 0) + { + fprintf(stderr, "Failed to mount nfs share : %s\n", nfs_get_error(_nfs)); + return 1; + } + return 0; + } + + std::vector NFS::listdir(std::string path, bool recursive = false, + bool with_file = true, bool with_dir = true, std::string pattern = "*") + { + int ret; + struct nfsdirent *nfsdirent; + struct nfsdir *nfsdir; + std::vector result; + if ((ret = nfs_opendir(_nfs, path.c_str(), &nfsdir)) != 0) + { + char error[1024]; + sprintf(error, "Failed to opendir(\"%s\"): %s\n", path.c_str(), + nfs_get_error(_nfs)); + free_context(); + throw PyNFSError(error); + } + + std::vector patterns; + pystring::split(pattern, patterns, ","); + + while ((nfsdirent = nfs_readdir(_nfs, nfsdir)) != NULL) + { + if (!strcmp(nfsdirent->name, ".") || !strcmp(nfsdirent->name, "..")) + { + continue; + } + std::string fpath = pystring::os::path::join_posix(path, nfsdirent->name); + if (recursive && (nfsdirent->mode & S_IFMT) == S_IFDIR) + { + + std::vector fpathes = listdir(fpath, recursive, with_file, with_dir, pattern); + for (auto p : fpathes) + { + for (auto ptn : patterns) + { + if (match(ptn.c_str(), p.c_str())) + { + result.push_back(p); + } + } + } + if (with_dir) + { + for (auto ptn : patterns) + { + if (match(ptn.c_str(), fpath.c_str())) + { + result.push_back(fpath); + } + } + } + } + else + { + if (with_file) + { + for (auto ptn : patterns) + { + if (match(ptn.c_str(), fpath.c_str())) + { + result.push_back(fpath); + } + } + } + } + } + nfs_closedir(_nfs, nfsdir); + return result; + } + + int NFS::mkdir(std::string path) + { + // https://github.com/sahlberg/libnfs/issues/181 + // Get mkdir permission with uid, nfs://address/nfs_share/?uid=0 + int ret; + if ((ret = nfs_mkdir(_nfs, path.c_str())) != 0) + { + // fprintf(stderr, "Failed to mkdir: %s\n", nfs_get_error(nfs)); + char error[1024]; + sprintf(error, "Failed to mkdir(\"%s\"): %s\n", path.c_str(), + nfs_get_error(_nfs)); + free_context(); + throw PyNFSError(error); + } + return ret; + } + + void NFS::rmtree(std::string path) + { + for (auto p : listdir(path)) + { + std::string fpath = pystring::os::path::join_posix(path, p); + if (isdir(fpath)) + { + rmtree(fpath); + } + else + { + remove(fpath); + } + } + rmdir(path); + } + + int NFS::rmdir(std::string path) + { + int ret; + if ((ret = nfs_rmdir(_nfs, path.c_str())) != 0) + { + char error[1024]; + sprintf(error, "Failed to rmdir(\"%s\"): %s\n", path.c_str(), + nfs_get_error(_nfs)); + free_context(); + throw PyNFSError(error); + } + return ret; + } + + int NFS::remove(std::string path) + { + int ret; + if ((ret = nfs_unlink(_nfs, path.c_str())) != 0) + { + char error[1024]; + sprintf(error, "Failed to unlink(\"%s\"): %s\n", path.c_str(), + nfs_get_error(_nfs)); + free_context(); + throw PyNFSError(error); + } + return ret; + } + + int NFS::chdir(std::string path) + { + int ret; + if ((ret = nfs_chdir(_nfs, path.c_str())) != 0) + { + char error[1024]; + sprintf(error, "Failed to chdir(\"%s\"): %s\n", path.c_str(), + nfs_get_error(_nfs)); + free_context(); + throw PyNFSError(error); + } + return 0; + } + + void NFS::makedirs(std::string path) + { + std::vector result; + pystring::split(pystring::replace(path, "\\", "/"), result, "/"); + std::string npath = "/"; + + for (auto p : result) + { + npath = pystring::os::path::join_posix(npath, p); + if (exists(npath)) + continue; + mkdir(npath); + } + } + + struct nfs_stat_64 NFS::fstat(std::string path) + { + struct nfs_stat_64 stat; + if (nfs_stat64(_nfs, path.c_str(), &stat) != 0) + { + // fprintf(stderr, "Failed to stat file : %s\n", nfs_get_error(nfs)); + char error[1024]; + sprintf(error, "Failed to stat file: %s\n", + nfs_get_error(_nfs)); + free_context(); + throw PyNFSError(error); + } + return stat; + } + + int NFS::chmod(std::string path, int mode) + { + int ret; + if ((ret = nfs_chmod(_nfs, path.c_str(), mode)) != 0) + { + //fprintf(stderr, "Failed to chmod(): %s\n", nfs_get_error(nfs)); + char error[1024]; + sprintf(error, "Failed to chmod(): %s\n", + nfs_get_error(_nfs)); + free_context(); + throw PyNFSError(error); + } + return ret; + } + + int NFS::chown(std::string path, int uid, int gid) + { + int ret; + if ((ret = nfs_chown(_nfs, path.c_str(), uid, gid)) != 0) + { + char error[1024]; + sprintf(error, "Failed to chown(): %s\n", + nfs_get_error(_nfs)); + free_context(); + throw PyNFSError(error); + } + return ret; + } + + bool NFS::exists(std::string path) + { + struct nfs_stat_64 stat; + if (nfs_stat64(_nfs, path.c_str(), &stat) != 0) + { + return false; + } + return true; + } + + bool NFS::isfile(std::string path) + { + struct nfs_stat_64 st = fstat(path); + py::object isreg = py::module_::import("stat").attr("S_ISREG")(st.nfs_mode); + return isreg.cast(); + } + + bool NFS::isdir(std::string path) + { + struct nfs_stat_64 st = fstat(path); + py::object isdir = py::module_::import("stat").attr("S_ISDIR")(st.nfs_mode); + return isdir.cast(); + } + + time_t NFS::getmtime(std::string path) + { + struct nfs_stat_64 st = fstat(path); + return st.nfs_mtime; + } + + time_t NFS::getatime(std::string path) + { + struct nfs_stat_64 st = fstat(path); + return st.nfs_atime; + } + + time_t NFS::getctime(std::string path) + { + struct nfs_stat_64 st = fstat(path); + return st.nfs_ctime; + } + + std::vector NFS::glob(std::string path, bool recursive) + { + return std::vector(); + } + + void NFS::set_uid(int uid) + { + nfs_set_uid(_nfs, uid); + } + + void NFS::set_gid(int gid) + { + nfs_set_gid(_nfs, gid); + } + + std::string NFS::getcwd() + { + const char *cwd[1024]; + nfs_getcwd(_nfs, cwd); + return std::string(*cwd); + } + + void NFS::free_context() + { + if (_url != NULL) + { + nfs_destroy_url(_url); + _url = NULL; + } + if (_nfs != NULL) + { + nfs_destroy_context(_nfs); + _nfs = NULL; + } + } + + int NFS::rename(std::string old_path, std::string new_path) + { + int ret; + if ((ret = nfs_rename(_nfs, old_path.c_str(), new_path.c_str())) != 0) + { + //fprintf(stderr, "Failed to rename(): %s\n", nfs_get_error(nfs)); + char error[1024]; + sprintf(error, "Failed to rename(\"%s\", \"%s\"): %s\n", + old_path.c_str(), new_path.c_str(), + nfs_get_error(_nfs)); + throw PyNFSError(error); + } + return ret; + } + + int NFS::symlink(std::string target, std::string linkname) + { + int ret; + if ((ret = nfs_symlink(_nfs, target.c_str(), linkname.c_str())) != 0) + { + char error[1024]; + sprintf(error, "Failed to symlink(\"%s\", \"%s\"): %s\n", + target.c_str(), linkname.c_str(), + nfs_get_error(_nfs)); + throw PyNFSError(error); + } + return ret; + } + + std::shared_ptr NFS::fopen(std::string path, std::string mode) + { + /*std::cout << getcwd() << std::endl; + if (!pystring::startswith(path, "nfs://")) { + char _path[1024]; + sprintf(_path, "nfs://%s%s/%s", + _url->server, _url->path, path.c_str()); + return NFSFH(_nfs, _path, mode); + } + else { + return NFSFH(_nfs, path, mode); + }*/ + struct file_context *ctx = new file_context; + if (ctx == NULL) + { + fprintf(stderr, "Failed to malloc file_context\n"); + return NULL; + } + ctx->nfs = _nfs; + ctx->url = _url; + return std::make_shared(ctx, path, mode); + } + + int NFS::put(std::string nfs_dst, std::string src, bool recursive = true, std::string pattern = "*") + { + std::cout << "put() : " << src << std::endl; + struct stat info; + std::string cwd = getcwd(); + std::string dst_path = pystring::os::path::join_posix(cwd, nfs_dst); + + // Destination must be directory + if (!pystring::endswith(dst_path, "/")) + { + dst_path = dst_path + "/"; + } + + if (!exists(dst_path)) + { + makedirs(dst_path); + } + + if (!fu::exists(src)) + { + fprintf(stderr, "src path \"%s\" not exists.", src.c_str()); + return 1; + } + + if (fu::isdir(src)) + { + putdir(nfs_dst, src, recursive, pattern); + } + else + { + std::string src_file_path = src; + std::string src_file = pystring::os::path::basename(src_file_path); + std::string dst_file_path = pystring::os::path::join_posix(dst_path, src_file); + putfile(dst_file_path, src_file_path); + } + + return 0; + } + + int NFS::putdir(std::string nfs_dst, std::string src, bool recursive, std::string pattern = "*") + { + // dst : ./dst + // src : C:/xx/Python27 + // C:/xx/Python27/a/a.pyd --> ./dst/Python27/a/a.pyd + + // dst : ./dst + // src : C:/xx/Python27/ + // C:/xx/Python27/a/a.pyd --> ./dst/a/a.pyd + + std::cout << "putdir() : " << src << std::endl; + std::vector src_pathes = fu::listdir(src, recursive, true, true, pattern); + for (auto src_path : src_pathes) + { + //std::string src_path_rel = pystring::replace(src_path, src, ""); + std::string src_path_rel = pystring::slice(src_path, src.length()); + if (!pystring::endswith(src, "/")) + { + + std::string src_dirname = pystring::os::path::dirname(src); + src_path_rel = pystring::slice(src_path, src_dirname.length()); + //src_path_rel = pystring::replace(src_path, src_dirname, ""); + } + if (pystring::startswith(src_path_rel, "/")) + { + src_path_rel = pystring::slice(src_path_rel, 1); + } + std::string dst_path = pystring::os::path::join_posix(nfs_dst, src_path_rel); + + if (!fu::exists(src_path)) + { + fprintf(stderr, "Failed to stat source file : %s", src_path.c_str()); + continue; + } + + if (fu::isdir(src_path)) + { + makedirs(dst_path); + } + else + { + putfile(dst_path, src_path); + } + } + return 0; + } + + int NFS::putfile(std::string nfs_dst, std::string src) + { + struct stat st; + off_t off; + ssize_t count; + + std::string src_file_path = src; + + //int fd = fu::open(src_file_path, O_RDONLY | O_BINARY); + int fd = fu::open(src_file_path, "rb"); + if (fd < 0) + { + fprintf(stderr, "Failed to open file for reading : \"%s\"\n", src_file_path.c_str()); + return 1; + } + + nfs_dst = pystring::replace(nfs_dst, "\\", "/"); + std::string dst_file_path = nfs_dst; + std::string dst_dir = pystring::os::path::dirname(dst_file_path); + + // Destination is directory + if (!exists(dst_dir)) + { + makedirs(dst_dir); + } + + auto nfsfh = fopen(dst_file_path, "wb+"); + + if (nfsfh == NULL) + { + fprintf(stderr, "Failed to open nfs file for write : \"%s\"\n", dst_file_path.c_str()); + fu::close(fd); + return 10; + } + + if (stat(src_file_path.c_str(), &st) != 0) + { + fprintf(stderr, "Failed to stat source file \"%s\"\n", src_file_path.c_str()); + fu::close(fd); + return 10; + } + + char* buf; + buf = (char*)malloc(BUFSIZE); + off = 0; + while (off < st.st_size) + { + count = (size_t)(st.st_size - off); + + if (count > BUFSIZE) + { + count = BUFSIZE; + } + + count = fu::pread(fd, buf, count, off); + if (count < 0) + { + fprintf(stderr, "Failed to read from source file\n"); + fu::close(fd); + nfsfh->free_context(); + return 10; + } + + count = nfsfh->pwrite(buf, count, off); + if (count < 0) + { + fprintf(stderr, "Failed to write to nfs target file\n"); + fu::close(fd); + nfsfh->fclose(); + return 10; + } + off += count; + } + + if (buf != NULL) + { + free(buf); + } + + printf("Copied %d bytes : %s\n", (int)off, dst_file_path.c_str()); + + fu::close(fd); + nfsfh->fclose(); + return 0; + } + + int NFS::getdir(std::string nfs_src, std::string dst, bool recursive=true, std::string pattern="*") + { + std::vector src_pathes = listdir(nfs_src, recursive, true, true, pattern); + + dst = pystring::os::path::normpath(dst); + + for (auto src_path : src_pathes) + { + //std::cout << src_path << std::endl; + //continue; + std::string src_path_rel = pystring::slice(src_path, nfs_src.length()); + if (!pystring::endswith(nfs_src, "/")) + { + std::string src_dirname = pystring::os::path::dirname(nfs_src); + std::cout << "getdir : src_dirname :" << src_dirname << std::endl; + src_path_rel = pystring::slice(src_path, src_dirname.length()); + } + + if (pystring::startswith(src_path_rel, "/")) + { + src_path_rel = pystring::slice(src_path_rel, 1); + } + + /*std::cout << "getdir : dst :" << dst << std::endl; + std::cout << "getdir : src :" << src << std::endl; + std::cout << "getdir : src_path :" << src_path << std::endl; + std::cout << "getdir : src_path_rel : " << src_path_rel << std::endl;*/ + + std::string dst_path = pystring::os::path::normpath(pystring::os::path::join(dst, src_path_rel)); + + if (!exists(src_path)) + { + fprintf(stderr, "Failed to stat nfs source file : %s", src_path.c_str()); + continue; + } + if (isdir(src_path)) + { + makedirs(dst_path); + } + else + { + getfile(src_path, dst_path); + } + //std::cout << "----" << std::endl; + } + return 0; + } + + int NFS::getfile(std::string nfs_src, std::string dst) + { + /*std::cout << "getfile : src : " << nfs_src << std::endl; + std::cout << "getfile : dst : " << dst << std::endl;*/ + //return 0; + off_t off; + ssize_t count; + + std::string src_file_path = nfs_src; + std::string src_dir = pystring::os::path::dirname(src_file_path); + + auto nfsfh = fopen(src_file_path, "rb"); + + if (nfsfh == NULL) + { + fprintf(stderr, "Failed to open nfs file for read : \"%s\"\n", src_file_path.c_str()); + return 10; + } + + dst = pystring::replace(dst, "\\", "/"); + std::string dst_file_path = dst; + std::string dst_dir = pystring::os::path::dirname(dst_file_path); + if (!fu::exists(dst_dir)) + { + fu::makedirs(dst_dir); + } + + int fd = fu::open(dst_file_path, "wb+"); + + char* buf; + buf = (char*)malloc(BUFSIZE); + off = 0; + auto st = nfsfh->fstat(); + while (off < st.nfs_size) + { + count = (size_t)(st.nfs_size - off); + + if (count > BUFSIZE) + { + count = BUFSIZE; + } + count = nfsfh->pread(buf, count, off); + if (count < 0) + { + fprintf(stderr, "Failed to read from source nfs file\n"); + fu::close(fd); + nfsfh->free_context(); + return 10; + } + count = fu::pwrite(fd, buf, count, off); + if (count < 0) + { + fprintf(stderr, "Failed to write to target file\n"); + fu::close(fd); + nfsfh->fclose(); + return 10; + } + off += count; + } + + if (buf != NULL) + { + free(buf); + } + + printf("Copied %d bytes : %s\n", (int)off, dst_file_path.c_str()); + + fu::close(fd); + nfsfh->fclose(); + + return 0; + } + + int NFS::get(std::string nfs_src, std::string dst, bool recursive=true, std::string pattern="*") + { + std::cout << "get() : " << nfs_src << std::endl; + std::string dst_path = dst; + if (!pystring::endswith(dst_path, "/")) + { + dst_path = dst_path + "/"; + } + if (!fu::exists(dst_path)) + { + fu::makedirs(dst_path); + } + + if (!exists(nfs_src)) + { + fprintf(stderr, "nfs src path \"%s\" not exists.\n", nfs_src.c_str()); + return 1; + } + + if (isdir(nfs_src)) + { + getdir(nfs_src, dst, recursive, pattern); + } + else + { + std::string src_file_path = nfs_src; + std::string src_file = pystring::os::path::basename(src_file_path); + std::string dst_file_path = pystring::os::path::join_posix(dst_path, src_file); + getfile(src_file_path, dst_file_path); + } + + return 0; + } + + void NFS::test() + { + set_uid(0); + set_gid(0); + if (!exists("./a")) + mkdir("./a"); + makedirs("./a/b/c"); + auto fn = fopen("./a/b/c/test.txt", "w+"); + for (int i = 0; i <= 1000; i++) + { + char buf[1024]; + sprintf(buf, "This is a test %d\n", i); + fn->fwrite(buf); + } + fn->fclose(); + + for (auto p : listdir("./a", true, true, "*")) + { + std::cout << p << std::endl; + } + //put("./", "C:/Python27"); + chmod("./a/b/c/test.txt", 0777); + chown("./a/b/c/test.txt", 1000, 1000); + get("./Python27/", "C:/test"); + //putfile("./ttt/", "C:/Python27/NEWS.txt"); + //putfile("./ttt/addColorSwitch.png", "C:/CGCG/projects/COOKIES/maya/2018/x64/icons/addColorSwitch.png"); + } +} \ No newline at end of file diff --git a/src/NFS.h b/src/NFS.h new file mode 100644 index 0000000..bc7f0db --- /dev/null +++ b/src/NFS.h @@ -0,0 +1,72 @@ +#pragma once + +#include "pynfs.h" + +namespace pynfs +{ + class NFSFH; + + class NFS + { + public: + NFS(); + NFS(std::string path); + ~NFS(); + int openurl(std::string path); + + std::vector listdir(std::string path, bool recursive, bool with_file, bool with_dir, std::string pattern); + + int mkdir(std::string path); + void rmtree(std::string path); + int rmdir(std::string path); + int remove(std::string path); + int chdir(std::string path); + void makedirs(std::string path); + int chmod(std::string path, int mode); + int chown(std::string path, int uid, int gid); + int rename(std::string old_path, std::string new_path); + int symlink(std::string target, std::string linkname); + + struct nfs_stat_64 fstat(std::string path); + bool exists(std::string path); + bool isfile(std::string path); + bool isdir(std::string path); + + time_t getmtime(std::string path); + time_t getatime(std::string path); + time_t getctime(std::string path); + + std::vector glob(std::string path, bool recursive); + std::shared_ptr fopen(std::string path, std::string mode); + + int putdir(std::string dst, std::string src, bool recursive, std::string pattern); + int putfile(std::string dst, std::string src); + int put(std::string nfs_dst, std::string src, bool recursive, std::string pattern); + + int getdir(std::string src, std::string dst, bool resursive, std::string pattern); + int getfile(std::string src, std::string dst); + int get(std::string nfs_src, std::string dst, bool recursive, std::string pattern); + + void test(); + + // void syncdir(std::string src_path, std::string dst_path); + + void set_uid(int uid); + void set_gid(int gid); + + std::string server() { return _url->server; }; + std::string path() { return _url->path; }; + std::string file() { return _url->file; }; + + std::string getcwd(); + + nfs_context *nfs() { return _nfs; }; + struct nfs_url *url() { return _url; }; + void free_context(); + + private: + nfs_context *_nfs; + nfs_url *_url; + }; + +} \ No newline at end of file diff --git a/src/NFSFH.cpp b/src/NFSFH.cpp new file mode 100644 index 0000000..9c50beb --- /dev/null +++ b/src/NFSFH.cpp @@ -0,0 +1,255 @@ +#include "NFS.h" +#include "NFSFH.h" + +#include "pystring.h" +#include + +namespace pynfs +{ + NFSFH::NFSFH(void* ctx, std::string path, std::string mode) + { + _nfs = NULL; + _nfsfh = NULL; + _url = NULL; + + _ctx = static_cast (ctx); + + if (_ctx == NULL) { + char error[1024]; + sprintf(error, "Failed to init file_context\n"); + throw PyNFSError(error); + } + else { + _nfs = _ctx->nfs; + _url = _ctx->url; + } + + if (_ctx->nfs == NULL) { + if (!pystring::startswith(path, "nfs://")) { + char error[1024]; + sprintf(error, "Failed to init context from path: %s\n", path.c_str()); + throw PyNFSError(error); + } + + auto nfs_ctx = NFS::NFS(path); + _nfs = nfs_ctx.nfs(); + _url = nfs_ctx.url(); + if (_nfs == NULL) { + char error[1024]; + sprintf(error, "Failed to init context"); + throw PyNFSError(error); + } + } + + /*if (_url == NULL) + { + if (!pystring::startswith(path, "nfs://")) { + char error[1024]; + sprintf(error, "Failed to init url from path: %s\n", path.c_str()); + throw PyNFSError(error); + } + else { + _url = nfs_parse_url_full(_nfs, path.c_str()); + if (_url == NULL) { + char error[1024]; + sprintf(error, "%s\n", nfs_get_error(_nfs)); + free_context(); + throw PyNFSError(error); + } + } + }*/ + + _binary = pystring::count(mode, "b") ? true : false; + _plus = pystring::endswith(mode, "+"); + _flag = 0; + if (pystring::startswith(mode, "r")) { + _flag = _plus ? O_RDWR : O_RDONLY; + if (pystring::startswith(mode, "rb")) { + _flag |= O_BINARY; + } + } + + if (pystring::startswith(mode, "w")) { + _flag = _plus ? O_RDWR : O_WRONLY; + _flag |= O_CREAT | O_TRUNC; + if(pystring::startswith(mode, "wb")) { + _flag |= O_BINARY; + } + } + + if (pystring::startswith(mode, "a")) { + _flag = _plus ? O_RDWR : O_WRONLY; + _flag |= O_CREAT | O_APPEND; + } + + if (nfs_open(_nfs, path.c_str(), _flag, &_nfsfh) != 0) { + if (_flag & O_CREAT) { + if (nfs_create(_nfs, path.c_str(), _flag, 0664, &_nfsfh) != 0) { + char error[1024]; + sprintf(error, "Failed to create file %s: %s\n", + _url->file, + nfs_get_error(_nfs)); + throw PyNFSError(error); + free_context(); + } + } + else { + char error[1024]; + sprintf(error, "Failed to open file %s: %s\n", + _url->file, + nfs_get_error(_nfs)); + free_context(); + throw PyNFSError(error); + } + } + _closed = false; + _need_flush = false; + _writing = _flag & (O_RDWR | O_WRONLY) ? true : false; + } + + NFSFH::NFSFH(std::string path, std::string mode) + { + if (!pystring::startswith(path, "nfs://")) { + char error[1024]; + sprintf(error, "Failed to init context from path: %s\n", path.c_str()); + throw PyNFSError(error); + } + } + + void NFSFH::fclose() + { + if (_need_flush) { + flush(); + } + nfs_close(_nfs, _nfsfh); + _closed = true; + } + + std::string NFSFH::fread(int size = -1) + { + if (size < 0) { + uint64_t pos = tell(); + struct nfs_stat_64 st; + nfs_fstat64(_nfs, _nfsfh, &st); + size = (int)(st.nfs_size - pos); + } + + char buf[BUFSIZE]; + int count; + count = nfs_read(_nfs, _nfsfh, sizeof(buf), buf); + if (count < 0) { + char error[1024]; + sprintf(error, "Failed to read(): %s\n", + nfs_get_error(_nfs)); + } + return std::string(buf).substr(0, count); + } + + int NFSFH::fwrite(char* data) + { + int ret = 0; + int count = nfs_write(_nfs, _nfsfh, strlen(data), data); + if (count < 0) { + char error[1024]; + sprintf(error, "Failed to write(): %s\n", + nfs_get_error(_nfs)); + ret = 1; + } + _need_flush = true; + return ret; + } + + void NFSFH::fseek(int64_t offset, int whence = SEEK_CUR) + { + uint64_t current_offset; + nfs_lseek(_nfs, _nfsfh, offset, whence, ¤t_offset); + } + + uint64_t NFSFH::fileno() + { + struct nfs_stat_64 st; + nfs_fstat64(_nfs, _nfsfh, &st); + return st.nfs_ino; + } + + nfs_stat_64 NFSFH::fstat() + { + struct nfs_stat_64 stat; + if (nfs_fstat64(_nfs, _nfsfh, &stat) != 0) { + char error[1024]; + sprintf(error, "Failed to stat file: %s\n", + nfs_get_error(_nfs)); + free_context(); + throw PyNFSError(error); + } + return stat; + } + + uint64_t NFSFH::tell() + { + uint64_t pos; + nfs_lseek(_nfs, _nfsfh, 0, SEEK_CUR, &pos); + return pos; + } + + void NFSFH::truncate(uint64_t offset = -1) + { + if (offset < 0) { + offset = tell(); + } + nfs_ftruncate(_nfs, _nfsfh, offset); + } + + void NFSFH::flush() + { + if (_closed) { + throw PyNFSError("I/O operation on closed file"); + return; + } + nfs_fsync(_nfs, _nfsfh); + _need_flush = false; + } + + const size_t NFSFH::pread(char* buf, size_t count, off_t off) + { + return nfs_pread(_nfs, _nfsfh, off, count, buf);; + } + + const size_t NFSFH::pwrite(char* buf, size_t count, off_t off) + { + return nfs_pwrite(_nfs, _nfsfh, off, count, buf); + } + + std::string NFSFH::getcwd() + { + const char* cwd[1024]; + nfs_getcwd(_nfs, cwd); + return std::string(*cwd); + } + + void NFSFH::error() + { + } + + void NFSFH::enter() + { + } + + void NFSFH::exit() + { + } + + const void NFSFH::free_context() + { + /*if (_fd != -1) { + close(_fd); + }*/ + if (_nfsfh != NULL) { + nfs_close(_nfs, _nfsfh); + } + if (_nfs != NULL) { + nfs_destroy_context(_nfs); + } + nfs_destroy_url(_url); + } +} \ No newline at end of file diff --git a/src/NFSFH.h b/src/NFSFH.h new file mode 100644 index 0000000..eea0a00 --- /dev/null +++ b/src/NFSFH.h @@ -0,0 +1,53 @@ +#pragma once + +#include "pynfs.h" + +namespace pynfs +{ + class NFS; + + class NFSFH + { + public: + NFSFH(void* ctx, std::string path, std::string mode); + NFSFH(std::string path, std::string mode); + + void fclose(); + int fwrite(char* data); + std::string fread(int size); + //char* fread(); + void fseek(int64_t offset, int whence); + uint64_t fileno(); + struct nfs_stat_64 fstat(); + uint64_t tell(); + void truncate(uint64_t offset); + void flush(); + + const size_t pread(char* buf, size_t count, off_t off); + const size_t pwrite(char* buf, size_t count, off_t off); + + nfs_context* nfs() { return _nfs; }; + struct nfs_url* url() { return _url; }; + struct nfsfh* nfsfh() { return _nfsfh; }; + + std::string getcwd(); + + void error(); + void enter(); + void exit(); + const void free_context(); + + private: + struct file_context* _ctx; + nfs_context* _nfs; + struct nfsfh* _nfsfh; + struct nfs_url* _url; + + int _flag; + bool _plus; + bool _binary; + bool _closed; + bool _need_flush; + bool _writing; + }; +} \ No newline at end of file diff --git a/src/fileutils.hpp b/src/fileutils.hpp new file mode 100644 index 0000000..7166a4c --- /dev/null +++ b/src/fileutils.hpp @@ -0,0 +1,200 @@ +#define _FILE_OFFSET_BITS 64 + +#include +#include +#include +#include +#include +#include +#include +#include +#include "pystring.h" + +#undef open +#undef close + +namespace fu +{ + bool exists(std::string path) + { + struct stat st; + if (stat(path.c_str(), &st) != 0) { + return false; + } + return true; + } + + void makedirs(std::string path) + { + std::vector result; + pystring::split(pystring::replace(path, "\\", "/"), result, "/"); + std::string npath = result[0]; + for (auto p : result) + { + if (p == result[0]) + continue; + npath = pystring::os::path::join_posix(npath, p); + if (exists(npath)) + continue; + mkdir(npath.c_str()); + } + } + bool isdir(std::string path) + { + struct stat st; + if (stat(path.c_str(), &st) != 0) { + return false; + } + + if (st.st_mode & S_IFDIR) + return true; + + return false; + } + + bool isfile(std::string path) + { + struct stat st; + if (stat(path.c_str(), &st) != 0) { + return false; + } + if (st.st_mode & S_IFREG) + return true; + + return false; + } + + time_t getmtime(std::string path) + { + struct stat st; + if (stat(path.c_str(), &st) != 0) { + return 0; + } + return st.st_mtime; + } + + time_t getatime(std::string path) + { + struct stat st; + if (stat(path.c_str(), &st) != 0) { + return 0; + } + return st.st_atime; + } + + time_t getctime(std::string path) + { + struct stat st; + if (stat(path.c_str(), &st) != 0) { + return 0; + } + return st.st_ctime; + } + + struct stat fstat(std::string path) { + struct stat st; + stat(path.c_str(), &st); + return st; + } + + std::vector listdir(std::string path, bool recursive=false, bool with_file=true, bool with_dir=true, std::string pattern="*") + { + std::vector result; + + if (!isdir(path)) { + fprintf(stderr, "Path is not directory : %s", path.c_str()); + return result; + } + + WIN32_FIND_DATA data; + std::string rpath = pystring::os::path::join(path, pattern); + HANDLE hFind = FindFirstFile(rpath.c_str(), &data); + if (hFind != INVALID_HANDLE_VALUE) + { + do { + if (pystring::startswith(data.cFileName, ".") || pystring::startswith(data.cFileName, "..")) + continue; + + std::string fpath = pystring::os::path::join(path, data.cFileName); + if (recursive && isdir(fpath)) { + std::vector fpaths = listdir(fpath, recursive, with_file, with_dir, pattern); + for (auto p : fpaths) { + result.push_back(p); + } + if (with_dir){ + result.push_back(fpath); + } + } + else { + if (with_file) { + result.push_back(fpath); + } + } + } while (FindNextFile(hFind, &data)); + FindClose(hFind); + } + return result; + } + + int open(std::string path, int flags) { + int fd = _open(path.c_str(), flags, 0660); + if (fd == -1){ + fprintf(stderr, "Fail to open %s\n", path.c_str()); + return fd; + } + return fd; + } + + int open(std::string path, std::string mode) + { + bool _binary = pystring::count(mode, "b") ? true : false; + bool _plus = pystring::endswith(mode, "+") ? true : false; + int _flags = 0; + if (pystring::startswith(mode, "r")) { + _flags = _plus ? O_RDWR : O_RDONLY; + if (pystring::startswith(mode, "rb")) { + _flags |= O_BINARY; + } + } + + if (pystring::startswith(mode, "w")) { + _flags = _plus ? O_RDWR : O_WRONLY; + _flags |= O_CREAT | O_TRUNC; + if (pystring::startswith(mode, "wb")) { + _flags |= O_BINARY; + } + } + + if (pystring::startswith(mode, "a")) { + _flags = _plus ? O_RDWR : O_WRONLY; + _flags |= O_CREAT | O_APPEND; + } + int fd = _open(path.c_str(), _flags, 0660); + if (fd == -1) { + fprintf(stderr, "Fail to open %s\n", path.c_str()); + return fd; + } + return fd; + } + + int close(int fd) + { + if (fd != -1){ + return _close(fd); + } + return 0; + } + + static ssize_t pread(int fd, char* buf, size_t count, off_t off) + { + lseek(fd, off, SEEK_SET); + return read(fd, buf, count); + } + + static ssize_t pwrite(int fd, char* buf, size_t count, off_t off) + { + lseek(fd, off, SEEK_SET); + return write(fd, buf, count); + } +} + diff --git a/src/pynfs.cpp b/src/pynfs.cpp new file mode 100644 index 0000000..e8d02eb --- /dev/null +++ b/src/pynfs.cpp @@ -0,0 +1,73 @@ +#include "NFS.h" +#include "NFSFH.h" + +PYBIND11_MODULE(_pynfs, m) +{ + m.doc() = "libnfs python binding"; + using namespace pybind11::literals; + py::class_(m, "NFS") + .def(py::init<>()) + .def(py::init(), py::arg("path")) + .def("openurl", &pynfs::NFS::openurl, "Open a NFS url (nfs://server/path)") + .def("listdir", &pynfs::NFS::listdir, "List directory contents.", + py::arg("path") = ".", + py::arg("recursive") = false, + py::arg("with_file") = true, + py::arg("with_dir") = true, + py::arg("pattern") = "*") + .def("mkdir", &pynfs::NFS::mkdir, "Make new directory.") + .def("makedirs", &pynfs::NFS::makedirs, "Make new directories recursively.") + .def("remove", &pynfs::NFS::remove, "Remove a file or directory.") + .def("rmtree", &pynfs::NFS::rmtree, "Remove a directory tree.") + .def("rmdir", &pynfs::NFS::rmdir, "Remove a directory.") + .def("chmod", &pynfs::NFS::chmod, "Change permission") + .def("chown", &pynfs::NFS::chown, "Change owner") + .def("exists", &pynfs::NFS::exists, "Determine file or directory exists.") + .def("isdir", &pynfs::NFS::isdir, "Determine path is directory") + .def("isfile", &pynfs::NFS::isfile, "Determine path is file") + .def("chdir", &pynfs::NFS::chdir, "Change current working directory (cwd)") + .def("getcwd", &pynfs::NFS::getcwd, "Get current working directory") + .def("open", &pynfs::NFS::fopen, "Open a nfs file handle") + .def("getatime", &pynfs::NFS::getatime, "Get access time") + .def("getmtime", &pynfs::NFS::getmtime, "Get modify time") + .def("getctime", &pynfs::NFS::getctime, "Get create time") + .def("set_uid", &pynfs::NFS::set_uid, "Set uid") + .def("set_gid", &pynfs::NFS::set_gid, "Set gid") + .def("put", &pynfs::NFS::put, "Put file or directory to nfs path", + py::arg("nfs_dst"), py::arg("src"), + py::arg("recursive") = false, + py::arg("pattern") = "*") + .def("get", &pynfs::NFS::get, "Get file or directory from nfs path", + py::arg("nfs_src"), py::arg("dst"), + py::arg("recursive") = false, py::arg("pattern") = "*") + .def("test", &pynfs::NFS::test); + + py::class_(m, "NFSFH") + .def(py::init()) + .def(py::init()) + .def("read", &pynfs::NFSFH::fread, + py::arg("size") = -1) + .def("write", &pynfs::NFSFH::fwrite) + .def("seek", &pynfs::NFSFH::fseek, + py::arg("offset"), py::arg("whence") = SEEK_CUR) + .def("turncate", &pynfs::NFSFH::truncate, + py::arg("offset") = -1) + .def("tell", &pynfs::NFSFH::tell) + .def("fileno", &pynfs::NFSFH::fileno) + .def("flush", &pynfs::NFSFH::flush) + .def("stat", &pynfs::NFSFH::fstat) + .def("close", &pynfs::NFSFH::fclose) + .def("__enter__", &pynfs::NFSFH::enter) + .def("__exit__", &pynfs::NFSFH::exit); + + static py::exception ex(m, "PyNFSError"); + py::register_exception_translator([](std::exception_ptr p) + { + try { + if (p) std::rethrow_exception(p); + } + catch (const pynfs::PyNFSError&e) { + // Set MyException as the active python error + ex(e.what()); + } }); +} diff --git a/src/pynfs.h b/src/pynfs.h new file mode 100644 index 0000000..2c3a0ee --- /dev/null +++ b/src/pynfs.h @@ -0,0 +1,40 @@ +#pragma once +#include +#include +#include +#include + +#ifdef WIN32 +#include +#pragma comment(lib, "ws2_32.lib") +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#define BUFSIZE 1024*1024 + +namespace py = pybind11; + +namespace pynfs +{ + struct file_context { + nfs_context* nfs; + nfs_url* url; + }; + + class PyNFSError : public std::exception + { + public: + explicit PyNFSError(const char* m) : message{ m } {} + const char* what() const noexcept override { return message.c_str(); } + private: + std::string message = ""; + }; +} \ No newline at end of file diff --git a/src/wildcards.hpp b/src/wildcards.hpp new file mode 100644 index 0000000..5742459 --- /dev/null +++ b/src/wildcards.hpp @@ -0,0 +1,28 @@ +#include +#include + +bool match(char const* needle, char const* haystack) { + for (; *needle != '\0'; ++needle) { + switch (*needle) { + case '?': + if (*haystack == '\0') + return false; + ++haystack; + break; + case '*': { + if (needle[1] == '\0') + return true; + size_t max = strlen(haystack); + for (size_t i = 0; i < max; i++) + if (match(needle + 1, haystack + i)) + return true; + return false; + } + default: + if (*haystack != *needle) + return false; + ++haystack; + } + } + return *haystack == '\0'; +} \ No newline at end of file