http://gmd20.blog.163.com/blog/static/16843923201002274341552/
Linux中的sg驱动 (“The Linux SCSI Generic (sg) Driver“ http://sg.danny.cz/sg/index.html) 是一个通用SCSI设备的模型,应用程序通过定义的“SG_IO ioctl ”可以像scsi磁盘等设备发送自定义的scsi指令集。 2.6版本的,这个ioctl 将在 block驱动层的 (http://lxr.linux.no/#linux+v2.6.32/block/scsi_ioctl.c 文件的
static int sg_io(struct request_queue * q, struct gendisk * bd_disk, struct sg_io_hdr * hdr, fmode_t mode)
函数中得到处理,然后组成 request发送的底层的 scsi驱动。
使用的关键数据结构在 /include/scsi/sg.h 里面定义
关于具体用法,可以在“The Linux SCSI Generic (sg) HOWTO http://tldp.org/HOWTO/SCSI-Generic-HOWTO/index.html” 找到一些说明,“sg3_utils(http://sg.danny.cz/sg/sg3_utils.html)”的源码包里面的代码很不错的参考例 子,提供了各种scsi指令的封装代码,简单的scsi指令测试也可以直接诶使用sg3_utils里面的命令就可以了。 更多信息可以参考http://sg.danny.cz/sg/index.html 站点。
scatter-gather DMA
是一些磁盘或者网络硬件的扩展特性,可以让硬件设备在一次DMA映射里面访问不连续的多块内存,利用这个特性,可以直接让用户空间的地址直接映射DMA上 面去,减少传输过程中的数据复制量,提高系统西能。 “ Understanding the linux kernel ”一书的“block device” 一章的开头部分有介绍。
如果构建多段的 scatter-gather scsi request,可以参考sg3_utils 中的example目录下的sg_iovec_tst 那个例子。我简单修改一下源代码,发送一个自定义的scatter-gather列表的read10 来读磁盘的scsi request到设备的代码。其中 单个request的扇区总数和 scatter-gather列表个数的限制和block驱动层数和scsi3层驱动的scsi_host等参数设置有关,scatter-gather 列表还可能依赖于硬件实现。
#include <unistd.h>
#include <signal.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include "sg_lib.h"
#include "sg_io_linux.h"
/* Test co de for D. Gilbert's extensions to the Linux OS SCSI generic ("sg")
device driver.
* Copyright (C) 2003-2007 D. Gilbert
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2, or (at your option)
* any later version.
This program will read a certain number of blocks of a given block size
from a given sg device node and write what is retrieved out to a
normal file. The purpose is to test the sg_iovec mechanism within the
sg_io_hdr structure.
Version 0.12 (20070121)
*/
#define ME "sg_iovec_tst: "
#define A_PRIME 509
#define IOVEC_ELEMS 2048
#define SENSE_BUFF_LEN 32
#define DEF_TIMEOUT 40000 /* 40,000 milliseconds */
struct sg_iovec iovec[IOVEC_ELEMS];
/* Returns 0 if everything ok */
int sg_read(int sg_fd, unsigned char * buff, int num_blocks, int from_block,
int bs)
{
unsigned char rdCmd[10] = {READ_10, 0, 0, 0, 0, 0, 0, 0, 0, 0};
unsigned char senseBuff[SENSE_BUFF_LEN];
struct sg_io_hdr io_hdr;
int dxfer_len = bs * num_blocks;
int k=0, pos =0, rem=0;
rdCmd[2] = (unsigned char)((from_block >> 24) & 0xff);
rdCmd[3] = (unsigned char)((from_block >> 16) & 0xff);
rdCmd[4] = (unsigned char)((from_block >> 8) & 0xff);
rdCmd[5] = (unsigned char)(from_block & 0xff);
rdCmd[7] = (unsigned char)((num_blocks >> 8) & 0xff);
rdCmd[8] = (unsigned char)(num_blocks & 0xff);
/*
for (k = 0, pos = 0, rem = dxfer_len; k < IOVEC_ELEMS; ++k) {
iovec[k].iov_base = buff + pos;
iovec[k].iov_len = (rem > A_PRIME) ? A_PRIME : rem;
if (rem <= A_PRIME)
break;
pos += A_PRIME;
rem -= A_PRIME;
}
*/
//create 32 x 509 scatter-gather list , so remain ">128kb da ta" was split into the second request.
for (k = 0; k < 32; ++k) {
iovec[k].iov_base = buff + pos;
iovec[k].iov_len = 509;
pos += 512;
}
//put the remain into 2 scatter-gather list , so we don't exceed maxnium 64 segments.
rem = dxfer_len - (k+1) * 509 ;
iovec[k++].iov_base = buff + pos;
iovec[k++].iov_len = (rem /2) -3;
pos += rem /2 ;
iovec[k++].iov_base = buff + pos;
iovec[k++].iov_len = rem - (rem /2) + 3;
if (k >= IOVEC_ELEMS) {
fprintf(stderr, "Can't fit dxfer_len=%d bytes in iovec\n", dxfer_len);
return -1;
}
memset(&io_hdr, 0, sizeof(sg_io_hdr_t));
io_hdr.interface_id = 'S';
io_hdr.cmd_len = sizeof(rdCmd);
io_hdr.cmdp = rdCmd;
io_hdr.dxfer_direction = SG_DXFER_FROM_DEV;
io_hdr.dxfer_len = dxfer_len;
io_hdr.iovec_count = k + 1;
io_hdr.dxferp = iovec;
io_hdr.mx_sb_len = SENSE_BUFF_LEN;
io_hdr.sbp = senseBuff;
io_hdr.timeout = DEF_TIMEOUT;
io_hdr.pack_id = from_block;
if (ioctl(sg_fd, SG_IO, &io_hdr)) {
perror("reading (SG_IO) on sg device, error");
return -1;
}
switch (sg_err_category3(&io_hdr)) {
case SG_LIB_CAT_CLEAN:
break;
case SG_LIB_CAT_RECOVERED:
fprintf(stderr, "Recovered error while reading block=%d, num=%d\n",
from_block, num_blocks);
break;
case SG_LIB_CAT_UNIT_ATTENTION:
fprintf(stderr, "Unit attention\n");
return -1;
default:
sg_chk_n_print3("reading", &io_hdr, 1);
return -1;
}
return 0;
}
int main(int argc, char * argv[])
{
int sg_fd, fd, res, j, m, dxfer_len;
unsigned int k, num;
int do_help = 0;
int blk_size = 512;
int count = 0;
char * sg_file_name = 0;
char * out_file_name = 0;
unsigned char * buffp;
for (j = 1; j < argc; ++j) {
if (0 == strncmp("-b=", argv[j], 3)) {
m = 3;
num = sscanf(argv[j] + m, "%d", &blk_size);
if ((1 != num) || (blk_size <= 0)) {
printf("Couldn't decode number after '-b' switch\n");
sg_file_name = 0;
break;
}
}
else if (0 == strncmp("-c=", argv[j], 3)) {
m = 3;
num = sscanf(argv[j] + m, "%d", &count);
if (1 != num) {
printf("Couldn't decode number after '-c' switch\n");
sg_file_name = 0;
break;
}
}
else if (0 == strcmp("-h", argv[j]))
do_help = 1;
else if (*argv[j] == '-') {
printf("Unrecognized switch: %s\n", argv[j]);
sg_file_name = 0;
break;
}
else if (NULL == sg_file_name)
sg_file_name = argv[j];
else
out_file_name = argv[j];
}
if ((NULL == sg_file_name) || (NULL == out_file_name) || (0 == count)) {
printf("Usage: sg_iovec_tst [-h] [-b=num] -c=num <generic_device> "
"<output_filename>\n");
printf(" where: -h this usage message\n");
printf(" -b=num block size (default 512 Bytes)\n");
printf(" -c=num count of blocks to transfer\n");
printf(" reads from <generic_device> and sends to <output_filename>\n");
return 1;
}
sg_fd = open(sg_file_name, O_RDONLY);
if (sg_fd < 0) {
perror(ME "sg device node open error");
return 1;
}
/* Don't worry, being very careful not to write to a none-sg file ... */
res = ioctl(sg_fd, SG_GET_VERSION_NUM, &k);
if ((res < 0) || (k < 30000)) {
printf(ME "not a sg device, or driver prior to 3.x\n");
return 1;
}
fd = open(out_file_name, O_WRONLY | O_CREAT, 0666);
if (fd < 0) {
perror(ME "output file open error");
return 1;
}
count = 512 -1 ; //don't exceed 256kb IO
blk_size = 512;
dxfer_len = count * blk_size; //最终生成的request的大小
const int inter_space= 1024;
buffp = (unsigned char *)malloc(dxfer_len + inter_space);
if (buffp) {
if (0 == sg_read(sg_fd, buffp, count, 0, blk_size)) {
if (write(fd, buffp, dxfer_len) < 0)
perror(ME "output write failed");
}
free(buffp);
}
res = close(fd);
if (res < 0) {
perror(ME "output file close error");
close(sg_fd);
return 1;
}
res = close(sg_fd);
if (res < 0) {
perror(ME "sg device close error");
return 1;
}
return 0;
}
static int sg_io(struct request_queue * q, struct gendisk * bd_disk, struct sg_io_hdr * hdr, fmode_t mode)
函数中得到处理,然后组成 request发送的底层的 scsi驱动。
使用的关键数据结构在 /include/scsi/sg.h 里面定义
83typedef struct sg_io_hdr 84{ 85 int interface_id; /* [i] 'S' for SCSI generic (required) */ 86 int dxfer_direction; /* [i] data transfer direction */ 87 unsigned char cmd_len; /* [i] SCSI command length ( <= 16 bytes) */ 88 unsigned char mx_sb_len; /* [i] max length to write to sbp */ 89 unsigned short iovec_count; /* [i] 0 implies no scatter gather */ 90 unsigned int dxfer_len; /* [i] byte count of da ta transfer */ 91 void __user *dxferp; /* [i], [*io] points to da ta transfer memory 92 or scatter gather list */ 93 unsigned char __user *cmdp; /* [i], [*i] points to command to perform */ 94 void __user *sbp; /* [i], [*o] points to sense_buffer memory */ 95 unsigned int timeout; /* [i] MAX_UINT->no timeout (unit: millisec) */ 96 unsigned int flags; /* [i] 0 -> default, see SG_FLAG... */ 97 int pack_id; /* [i->o] unused internally (normally) */ 98 void __user * usr_ptr; /* [i->o] unused internally */ 99 unsigned char status; /* [o] scsi status */ 100 unsigned char masked_status;/* [o] shifted, masked scsi status */ 101 unsigned char msg_status; /* [o] messaging level da ta (optional) */ 102 unsigned char sb_len_wr; /* [o] byte count actually written to sbp */ 103 unsigned short host_status; /* [o] errors from host adapter */ 104 unsigned short driver_status;/* [o] errors from software driver */ 105 int resid; /* [o] dxfer_len - actual_transferred */ 106 unsigned int duration; /* [o] time taken by cmd (unit: millisec) */ 107 unsigned int info; /* [o] auxiliary information */ 108} sg_io_hdr_t; /* 64 bytes long (on i386) */
关于具体用法,可以在“The Linux SCSI Generic (sg) HOWTO http://tldp.org/HOWTO/SCSI-Generic-HOWTO/index.html” 找到一些说明,“sg3_utils(http://sg.danny.cz/sg/sg3_utils.html)”的源码包里面的代码很不错的参考例 子,提供了各种scsi指令的封装代码,简单的scsi指令测试也可以直接诶使用sg3_utils里面的命令就可以了。 更多信息可以参考http://sg.danny.cz/sg/index.html 站点。
scatter-gather DMA
是一些磁盘或者网络硬件的扩展特性,可以让硬件设备在一次DMA映射里面访问不连续的多块内存,利用这个特性,可以直接让用户空间的地址直接映射DMA上 面去,减少传输过程中的数据复制量,提高系统西能。 “ Understanding the linux kernel ”一书的“block device” 一章的开头部分有介绍。
如果构建多段的 scatter-gather scsi request,可以参考sg3_utils 中的example目录下的sg_iovec_tst 那个例子。我简单修改一下源代码,发送一个自定义的scatter-gather列表的read10 来读磁盘的scsi request到设备的代码。其中 单个request的扇区总数和 scatter-gather列表个数的限制和block驱动层数和scsi3层驱动的scsi_host等参数设置有关,scatter-gather 列表还可能依赖于硬件实现。
#include <unistd.h>
#include <signal.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include "sg_lib.h"
#include "sg_io_linux.h"
/* Test co
device driver.
* Copyright (C) 2003-2007 D. Gilbert
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2, or (at your option)
* any later version.
This program will read a certain number of blocks of a given block size
from a given sg device node and write what is retrieved out to a
normal file. The purpose is to test the sg_iovec mechanism within the
sg_io_hdr structure.
Version 0.12 (20070121)
*/
#define ME "sg_iovec_tst: "
#define A_PRIME 509
#define IOVEC_ELEMS 2048
#define SENSE_BUFF_LEN 32
#define DEF_TIMEOUT 40000 /* 40,000 milliseconds */
struct sg_iovec iovec[IOVEC_ELEMS];
/* Returns 0 if everything ok */
int sg_read(int sg_fd, unsigned char * buff, int num_blocks, int from_block,
int bs)
{
unsigned char rdCmd[10] = {READ_10, 0, 0, 0, 0, 0, 0, 0, 0, 0};
unsigned char senseBuff[SENSE_BUFF_LEN];
struct sg_io_hdr io_hdr;
int dxfer_len = bs * num_blocks;
int k=0, pos =0, rem=0;
rdCmd[2] = (unsigned char)((from_block >> 24) & 0xff);
rdCmd[3] = (unsigned char)((from_block >> 16) & 0xff);
rdCmd[4] = (unsigned char)((from_block >> 8) & 0xff);
rdCmd[5] = (unsigned char)(from_block & 0xff);
rdCmd[7] = (unsigned char)((num_blocks >> 8) & 0xff);
rdCmd[8] = (unsigned char)(num_blocks & 0xff);
/*
for (k = 0, pos = 0, rem = dxfer_len; k < IOVEC_ELEMS; ++k) {
iovec[k].iov_base = buff + pos;
iovec[k].iov_len = (rem > A_PRIME) ? A_PRIME : rem;
if (rem <= A_PRIME)
break;
pos += A_PRIME;
rem -= A_PRIME;
}
*/
//create 32 x 509 scatter-gather list , so remain ">128kb da
for (k = 0; k < 32; ++k) {
iovec[k].iov_base = buff + pos;
iovec[k].iov_len = 509;
pos += 512;
}
//put the remain into 2 scatter-gather list , so we don't exceed maxnium 64 segments.
rem = dxfer_len - (k+1) * 509 ;
iovec[k++].iov_base = buff + pos;
iovec[k++].iov_len = (rem /2) -3;
pos += rem /2 ;
iovec[k++].iov_base = buff + pos;
iovec[k++].iov_len = rem - (rem /2) + 3;
if (k >= IOVEC_ELEMS) {
fprintf(stderr, "Can't fit dxfer_len=%d bytes in iovec\n", dxfer_len);
return -1;
}
memset(&io_hdr, 0, sizeof(sg_io_hdr_t));
io_hdr.interface_id = 'S';
io_hdr.cmd_len = sizeof(rdCmd);
io_hdr.cmdp = rdCmd;
io_hdr.dxfer_direction = SG_DXFER_FROM_DEV;
io_hdr.dxfer_len = dxfer_len;
io_hdr.iovec_count = k + 1;
io_hdr.dxferp = iovec;
io_hdr.mx_sb_len = SENSE_BUFF_LEN;
io_hdr.sbp = senseBuff;
io_hdr.timeout = DEF_TIMEOUT;
io_hdr.pack_id = from_block;
if (ioctl(sg_fd, SG_IO, &io_hdr)) {
perror("reading (SG_IO) on sg device, error");
return -1;
}
switch (sg_err_category3(&io_hdr)) {
case SG_LIB_CAT_CLEAN:
break;
case SG_LIB_CAT_RECOVERED:
fprintf(stderr, "Recovered error while reading block=%d, num=%d\n",
from_block, num_blocks);
break;
case SG_LIB_CAT_UNIT_ATTENTION:
fprintf(stderr, "Unit attention\n");
return -1;
default:
sg_chk_n_print3("reading", &io_hdr, 1);
return -1;
}
return 0;
}
int main(int argc, char * argv[])
{
int sg_fd, fd, res, j, m, dxfer_len;
unsigned int k, num;
int do_help = 0;
int blk_size = 512;
int count = 0;
char * sg_file_name = 0;
char * out_file_name = 0;
unsigned char * buffp;
for (j = 1; j < argc; ++j) {
if (0 == strncmp("-b=", argv[j], 3)) {
m = 3;
num = sscanf(argv[j] + m, "%d", &blk_size);
if ((1 != num) || (blk_size <= 0)) {
printf("Couldn't decode number after '-b' switch\n");
sg_file_name = 0;
break;
}
}
else if (0 == strncmp("-c=", argv[j], 3)) {
m = 3;
num = sscanf(argv[j] + m, "%d", &count);
if (1 != num) {
printf("Couldn't decode number after '-c' switch\n");
sg_file_name = 0;
break;
}
}
else if (0 == strcmp("-h", argv[j]))
do_help = 1;
else if (*argv[j] == '-') {
printf("Unrecognized switch: %s\n", argv[j]);
sg_file_name = 0;
break;
}
else if (NULL == sg_file_name)
sg_file_name = argv[j];
else
out_file_name = argv[j];
}
if ((NULL == sg_file_name) || (NULL == out_file_name) || (0 == count)) {
printf("Usage: sg_iovec_tst [-h] [-b=num] -c=num <generic_device> "
"<output_filename>\n");
printf(" where: -h this usage message\n");
printf(" -b=num block size (default 512 Bytes)\n");
printf(" -c=num count of blocks to transfer\n");
printf(" reads from <generic_device> and sends to <output_filename>\n");
return 1;
}
sg_fd = open(sg_file_name, O_RDONLY);
if (sg_fd < 0) {
perror(ME "sg device node open error");
return 1;
}
/* Don't worry, being very careful not to write to a none-sg file ... */
res = ioctl(sg_fd, SG_GET_VERSION_NUM, &k);
if ((res < 0) || (k < 30000)) {
printf(ME "not a sg device, or driver prior to 3.x\n");
return 1;
}
fd = open(out_file_name, O_WRONLY | O_CREAT, 0666);
if (fd < 0) {
perror(ME "output file open error");
return 1;
}
count = 512 -1 ; //don't exceed 256kb IO
blk_size = 512;
dxfer_len = count * blk_size; //最终生成的request的大小
const int inter_space= 1024;
buffp = (unsigned char *)malloc(dxfer_len + inter_space);
if (buffp) {
if (0 == sg_read(sg_fd, buffp, count, 0, blk_size)) {
if (write(fd, buffp, dxfer_len) < 0)
perror(ME "output write failed");
}
free(buffp);
}
res = close(fd);
if (res < 0) {
perror(ME "output file close error");
close(sg_fd);
return 1;
}
res = close(sg_fd);
if (res < 0) {
perror(ME "sg device close error");
return 1;
}
return 0;
}