linux读写二进制流,linux读写二进制大文件

最新推荐文章于 2023-04-11 22:05:50 发布

藏在暗处的X先生

最新推荐文章于 2023-04-11 22:05:50 发布

阅读量671

点赞数

文章标签： linux读写二进制流

https://raw.githubusercontent.com/ceph/ceph/master/src/common/safe_io.c

一般而言，用pwrite和pread读写磁盘文件不需要用循环，但是在读写超大的文件时就一定需要循环，保证正确读写

// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-

// vim: ts=8 sw=2 smarttab

* Ceph - scalable distributed file system

* This is free software; you can redistribute it and/or

* modify it under the terms of the GNU Lesser General Public

* License version 2.1, as published by the Free Software

* Foundation. See file COPYING.

#define _XOPEN_SOURCE 500

#include

#include "common/safe_io.h"

#include "include/compat.h"

ssize_t safe_read(int fd, void *buf, size_t count)

{

size_t cnt = 0;

while (cnt < count) {

ssize_t r = read(fd, buf, count - cnt);

if (r <= 0) {

if (r == 0) {

// EOF

return cnt;

}

if (errno == EINTR)

continue;

return -errno;

}

cnt += r;

buf = (char *)buf + r;

}

return cnt;

}

ssize_t safe_read_exact(int fd, void *buf, size_t count)

{

ssize_t ret = safe_read(fd, buf, count);

if (ret < 0)

return ret;

if ((size_t)ret != count)

return -EDOM;

return 0;

}

ssize_t safe_write(int fd, const void *buf, size_t count)

{

while (count > 0) {

ssize_t r = write(fd, buf, count);

if (r < 0) {

if (errno == EINTR)

continue;

return -errno;

}

count -= r;

buf = (char *)buf + r;

}

return 0;

}

ssize_t safe_pread(int fd, void *buf, size_t count, off_t offset)

{

size_t cnt = 0;

char *b = (char*)buf;

while (cnt < count) {

ssize_t r = pread(fd, b + cnt, count - cnt, offset + cnt);

if (r <= 0) {

if (r == 0) {

// EOF

return cnt;

}

if (errno == EINTR)

continue;

return -errno;

}

cnt += r;

}

return cnt;

}

ssize_t safe_pread_exact(int fd, void *buf, size_t count, off_t offset)

{

ssize_t ret = safe_pread(fd, buf, count, offset);

if (ret < 0)

return ret;

if ((size_t)ret != count)

return -EDOM;

return 0;

}

ssize_t safe_pwrite(int fd, const void *buf, size_t count, off_t offset)

{

while (count > 0) {

ssize_t r = pwrite(fd, buf, count, offset);

if (r < 0) {

if (errno == EINTR)

continue;

return -errno;

}

count -= r;

buf = (char *)buf + r;

offset += r;

}

return 0;

}

#ifdef CEPH_HAVE_SPLICE

ssize_t safe_splice(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out,

size_t len, unsigned int flags)

{

size_t cnt = 0;

while (cnt < len) {

ssize_t r = splice(fd_in, off_in, fd_out, off_out, len - cnt, flags);

if (r <= 0) {

if (r == 0) {

// EOF

return cnt;

}

if (errno == EINTR)

continue;

return -errno;

}

cnt += r;

}

return cnt;

}

ssize_t safe_splice_exact(int fd_in, loff_t *off_in, int fd_out,

loff_t *off_out, size_t len, unsigned int flags)

{

ssize_t ret = safe_splice(fd_in, off_in, fd_out, off_out, len, flags);

if (ret < 0)

return ret;

if ((size_t)ret != len)

return -EDOM;

return 0;

}

#endif

int safe_write_file(const char *base, const char *file,

const char *val, size_t vallen)

{

int ret;

char fn[PATH_MAX];

char tmp[PATH_MAX];

int fd;

// does the file already have correct content?

char oldval[80];

ret = safe_read_file(base, file, oldval, sizeof(oldval));

if (ret == (int)vallen && memcmp(oldval, val, vallen) == 0)

return 0; // yes.

snprintf(fn, sizeof(fn), "%s/%s", base, file);

snprintf(tmp, sizeof(tmp), "%s/%s.tmp", base, file);

fd = open(tmp, O_WRONLY|O_CREAT|O_TRUNC, 0644);

if (fd < 0) {

ret = errno;

return -ret;

}

ret = safe_write(fd, val, vallen);

if (ret) {

VOID_TEMP_FAILURE_RETRY(close(fd));

return ret;

}

ret = fsync(fd);

if (ret < 0) ret = -errno;

VOID_TEMP_FAILURE_RETRY(close(fd));

if (ret < 0) {

unlink(tmp);

return ret;

}

ret = rename(tmp, fn);

if (ret < 0) {

ret = -errno;

unlink(tmp);

return ret;

}

fd = open(base, O_RDONLY);

if (fd < 0) {

ret = -errno;

return ret;

}

ret = fsync(fd);

if (ret < 0) ret = -errno;

VOID_TEMP_FAILURE_RETRY(close(fd));

return ret;

}

int safe_read_file(const char *base, const char *file,

char *val, size_t vallen)

{

char fn[PATH_MAX];

int fd, len;

snprintf(fn, sizeof(fn), "%s/%s", base, file);

fd = open(fn, O_RDONLY);

if (fd < 0) {

return -errno;

}

len = safe_read(fd, val, vallen);

if (len < 0) {

VOID_TEMP_FAILURE_RETRY(close(fd));

return len;

}

// close sometimes returns errors, but only after write()

VOID_TEMP_FAILURE_RETRY(close(fd));

return len;

}

下面是自己写的代码，为了验证read/writeBlock的正确性，分两批写入文件，然后又分两批读入文件，实际工程中只要一次读写就行了

#include

using namespace std;

bool writeBlock(int fd, int64_t &fdOft,int8_t *arr, int64_t len) {

int64_t num = 0;

int64_t oft = 0;

int8_t *arr1 = (int8_t*)arr;

while (oft < len) {

num = pwrite(fd, arr1+oft, len-oft, fdOft+oft);

if (num == -1) {

return false;

}

oft += num;

}

fdOft += oft;

return true;

}

bool readBlock(int fd, int64_t &fdOft, void *arr, int64_t len) {

int64_t num = 0;

int64_t oft = 0;

int8_t *b = (int8_t*)arr;

while (oft < len) {

num = pread(fd, b + oft, len - oft, fdOft + oft);

if (num <= 0) {

if (num == 0) {

if (oft != len) return false;

else return true;

}

if (errno == EINTR) {

continue;//the call was interrupted by a signal befor any data was read

}

return false;

}

oft += num;

}

fdOft += oft;

return true;

}

int main(int argc, char *argv[]) {

int64_t len = atol(argv[1]);

cout << "len = "<

int fd = open(argv[2], O_RDWR|O_CREAT|O_TRUNC, S_IRWXU);

if (fd == -1) {

cout << "not open\n";

return -1;

}

int64_t total = sizeof(double)*len;

cout << "total = " << total <

double *arr = (double*)malloc(total);

if (arr == NULL) {

cout << "malloc err\n";

return -1;

}

arr[0] = 0.001;

for (int64_t i = 1; i < len; ++i) {

arr[i] = arr[i-1] + 0.001;

}

//int64_t num = pwrite(fd,arr,total,0);

int64_t fdOft = 0;

int64_t first = total/2;

int64_t second = total - first;

bool flag = writeBlock(fd, fdOft,(int8_t*)arr, first);

if (!flag) {

cout << "first no\n";

}

else {

cout << "first yes\n";

}

flag = writeBlock(fd, fdOft, (int8_t*)arr + first, second);

//cout << num<

if (!flag) {

cout << "second no\n";

}

else {

cout << "second yes\n";

}

close(fd);

int fdr = open(argv[2], O_RDONLY);

double *arr1 = (double*)malloc(total);

if (arr1 == NULL) {

cout << "malloc error\n";

return -1;

}

int64_t readOft = 0;

if (readBlock(fdr, readOft, arr1, first)) {

cout << "first read yes\n";

}

if (readBlock(fdr, readOft, (int8_t*)arr1 + first, second)) {

cout << "second read yes\n";

}

flag = true;

for (int64_t i = 0; i < len; ++i) {

if (arr1[i] != arr[i]) {

flag = false;

cout << i << " no equal\n";

}

if (flag) cout << "equal\n";

close(fdr);

return 1;

}

藏在暗处的X先生

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
linux读写二进制流,linux读写二进制大文件

https://raw.githubusercontent.com/ceph/ceph/master/src/common/safe_io.c一般而言，用pwrite和pread读写磁盘文件不需要用循环，但是在读写超大的文件时就一定需要循环，保证正确读写// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-// ...
复制链接

扫一扫