存储之磁盘读写-----基于scsi cmd和bio

            这几天做了一个抽象磁盘驱动的例子，直接提供用户ioctl接口，把用户态buf（页对齐）写入（读取）到磁盘，并使用了两个方案做性能测试对比：一种scsi层，一种块层；

基于scsi   cdb  读写硬盘-----------裸盘读写

#include <linux/fs.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/cdev.h>
#include <linux/device.h>
#include <asm/uaccess.h>
#include <linux/semaphore.h>
#include <linux/spinlock.h>
#include <linux/delay.h>
#include <linux/scatterlist.h>
#include <scsi/scsi.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_transport.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_dbg.h>
#include <scsi/scsi_eh.h>
#include <linux/list.h>
#define NAME "disk_test_io"
//ioctrl cmd
#define DISK_OPEN  1000
#define DISK_READ  1001
#define DISK_WRITE  1002
#define DISK_DONE  1003
#define DISK_SYNC   1004
#define MAX_FREQUEUE_COUNT  1024
#define MAX_DONEQUEUE_COUNT 16
//erro no
#define ERR_NO_DEVICE   -1
#define ERR_BUSY    -2
#define ERR_PARAM   -3
typedef unsigned int uint32; 
typedef unsigned long long uint64; 
typedef unsigned char uint8;
static int chardev_var = 0;
//static int chardev_count = 0;
static struct semaphore sem;
static spinlock_t spin;
static struct cdev diskio_cdev;
static struct class *diskio_class;
static struct device *diskio_device;
static int device_major = 0;
struct scsi_device *g_sdev[1024]={NULL};
struct deviceid {
    uint32 hostnum;
 uint32 channel;
 uint32 id;
 uint32 lun;
 int fd;
};
struct iocmd_head{
    uint32 fd; //sdev[index]  --> index = fd
 char *buf;
 uint32 len;     //  x*4k
 uint64 offset;  //  x*512   sector
 uint64 seq_num; 
};
struct cmd_result{
 
 int result;
 uint64 seq_num;
 struct scsi_sense_hdr sshr;
 char cdb[10];
int nr_page;
 struct scatterlist sgl[128];
 struct page* pages[128];
 struct list_head free_list;
 struct list_head done_list;
};
struct queue_head{
 spinlock_t lock; 
 struct list_head list;
 int count;
};
struct queue_head free_queue_head;
struct queue_head done_queue_head;
DECLARE_COMPLETION_ONSTACK(com);
#define del_queue(head, member) ({   \
    struct cmd_result *pos,*n; \
    if(!list_empty(&head.list)){  \
    spin_lock(&head.lock); \
    list_for_each_entry_safe(pos, n, &head.list, member)        \
    kfree(pos); \
    head.count=0;\
    spin_unlock(&head.lock); \
    } \
   })
#define get_one_node(head, member) ({  \
         struct cmd_result *p = NULL;\
         spin_lock(&(head).lock);\
         if(list_empty(&head.list)) {\
          spin_unlock(&head.lock);\
          p = NULL;\
         }\
         else {\
         p = container_of(head.list.next, struct cmd_result, member);\
         list_del(head.list.next);\
         (head).count--;\
         spin_unlock(&(head).lock);}\
         p;\
    })  
         
#define add_one_node(new, head, member) ({ \
     spin_lock(&head.lock);\
     head.count++;\
     list_add(&(new)->member,&head.list);\
     spin_unlock(&head.lock);\
    })         
void init_queue_head(struct queue_head * q)
{
	 INIT_LIST_HEAD(&q->list);
	 q->count = 0;
	 spin_lock_init(&q->lock);
}
void init_free_queue(void)
{
	 int i;
	 struct cmd_result *res = NULL;
	 for(i = 0; i < MAX_FREQUEUE_COUNT; i++) {
		  res = kzalloc(sizeof(struct cmd_result), GFP_KERNEL);
		  if(!res){printk("kzalloc error ! \n");}
		  add_one_node(res, free_queue_head, free_list);
	 }
}
int io_open(struct deviceid *dev_id){
     
  struct Scsi_Host *shost=NULL;
  struct scsi_device *sdev=NULL;
  int index=0;
  printk("kernel dev_id struct { %d, %d,%d,%d }\n",dev_id->hostnum,dev_id->channel,dev_id->id,dev_id->lun);
  shost=scsi_host_lookup(dev_id->hostnum);
  if(shost){
	sdev=scsi_device_lookup(shost,dev_id->channel,dev_id->id,dev_id->lun);
	  if(sdev){
		 for(index=0;index<1024;index++){                //this sdev already exists  
			if(g_sdev[index]&&(g_sdev[index]==sdev))
			return index;
		 }
		 for(index=0;index<1024;index++){               //this sdev is new ,so add to g_sdev
			if(g_sdev[index]==NULL){
			 g_sdev[index]=sdev;
			 printk("the new sdev=0x%p,\n",sdev);
			 return index;
			}
		 }
	  }else{
	   printk("scsi device %d : %d: %d :%d not find ! --------%s--%d ! ",dev_id->hostnum,dev_id->channel,dev_id->id,dev_id->lun, __FUNCTION__,__LINE__);
	  }
  }
  else{
	printk("scsi host %d not find !---%s--%d --!\n", dev_id->hostnum,__FUNCTION__,__LINE__);
  }
  return  ERR_NO_DEVICE;   //ERR_NO_DEVICE = -1  will be change
}

void diskio_done(struct scsi_cmnd *scmd ){
	struct cmd_result *res = (struct cmd_result*)scmd->request;
	printk("diskio_done !\n");
	if (res == NULL)
		goto out;
	res->result = scmd->result;
	for(i=0;i<res->nr_page;i++)
		page_cache_release(res->pages[i]);
	scsi_normalize_sense(scmd->sense_buffer, sizeof(struct scsi_sense_hdr), &res->sshr);
	printk("scsi done seq num  %lld\n", res->seq_num);
	printk("response_code %d, sense_key %d, asc %d, ascq %d\n", res->sshr.response_code, res->sshr.sense_key, res->sshr.asc, res->sshr.ascq);
	add_one_node(res, done_queue_head, done_list);
	printk("execute scsi cmd return result=%x  \n",res->result);
	printk("frequeue count = %d,  done queue count == %d\n", free_queue_head.count, done_queue_head.count);
out:
	complete(&com);
	scsi_put_command(scmd);
}
static int do_io(int cmd , int fd,  char * buf, uint32 len, uint64 offset, uint64 seq_num)
{
	int ret=0;
	int nr=0;
	int i=0;
	unsigned long flags;
	int cdb_len=0;
	struct scsi_cmnd *scmd;
	unsigned char *cdb;
	struct page ** pages;
	struct scatterlist *sg;
	uint32 offset_h=(uint32)((offset&0xffffffff00000000)>>32);
	uint32 offset_l=(uint32)((offset&0x00000000ffffffff));
	int nr_pages=((uint64)buf+len+PAGE_SIZE-1)/PAGE_SIZE-(uint64)buf/PAGE_SIZE;
 
	struct cmd_result *res = get_one_node(free_queue_head, free_list);
	if (!res) return ERR_BUSY; //ERR_BUSY = -2  
	cdb= res->cdb;
	pages= res->pages;
	sg= res->sgl;;
	res->seq_num = seq_num;
	res->nr_page=nr_pages;
	if(!g_sdev[fd]){
		printk("g_sdev \n"); 
		return ERR_PARAM;  ERR_PARAM= -3
	}
	if((len%512) || ( len > 128*512)) {
		printk("buf length should be x*512 !");
		return ERR_PARAM;  //ERR_PARAM= -3
	};
	if(offset < 0xffffffff){
		if(cmd==DISK_READ) {
			cdb[0]=0x28;    //operation code 28  read(10)
		}else if(cmd==DISK_WRITE){
			cdb[0]=0x2A;    //operation code 2A  write(10)
		}    
	   cdb[1]=0x0;        //FUA
	   cdb[2]=(uint8)((offset&0xff000000)>>24);      //LBA   logical block bufess cdb[2-5] offset
	   cdb[3]=(uint8)((offset&0x00ff0000)>>16);
	   cdb[4]=(uint8)((offset&0x0000ff00)>>8);
	   cdb[5]=(uint8)( offset&0x000000ff);
	   cdb[6]=0;   //group number
	   cdb[7]=(uint8)((((len+512-1)/512)&0x0000ff00) >>8);         //transfer number of sector   cdb[7-8]
	   cdb[8]=(uint8)(((len+512-1)/512)&0x000000ff);
	   cdb[9]=0;                     //control
	   cdb_len=10;
	   for(i=0;i<10;i++)
	   printk("cdb[%d]= 0x%x ==%d !\n",i,cdb[i],cdb[i]);
	}else{   //offset >0xffffffff
	   if(cmd==DISK_READ) {
			cdb[0]=0x88;  //operation code 28  read(16)
	   }else if(cmd==DISK_WRITE){
			cdb[0]=0x8A; //operation code 8A  write(16)
	   }    
	   cdb[1]=0x0;   //FUA
	   cdb[2]=(uint8)((offset_h&0xff000000)>>24);      //LBA   logical block bufess cdb[2-9] offset
	   cdb[3]=(uint8)((offset_h&0x00ff0000)>>16);
	   cdb[4]=(uint8)((offset_h&0x0000ff00)>>8);
	   cdb[5]=(uint8)( offset_h&0x000000ff);
	   cdb[6]=(uint8)((offset_l&0xff000000)>>24);
	   cdb[7]=(uint8)((offset_l&0x00ff0000)>>16);     
	   cdb[8]=(uint8)((offset_l&0x0000ff00)>>8);
	   cdb[9]=(uint8)( offset_l&0x000000ff);                    
	   cdb[10]=(uint8)((((len+512-1)/512)&0xff000000) >>24);  //transfer number of sector   cdb[10-13]
	   cdb[11]=(uint8)((((len+512-1)/512)&0x00ff0000)>>16);
	   cdb[12]=(uint8)((((len+512-1)/512)&0x0000ff00)>>8);
	   cdb[13]=(uint8)(((len+512-1)/512)&0x000000ff);
	   cdb[14]=0;            //group number
	   cdb[15]=0;            //control
	   cdb_len=16;
	   for(i=0;i<16;i++)
	   printk("cdb[%d]= 0x%x ==%d !\n",i,cdb[i],cdb[i]);
	}
	printk("enter the do_io :cmd=%d , fd= %d , buf= 0X%p ,len =%d, offset= %lld  \n",cmd,fd, buf,len,offset);
	down_read(¤t->mm->mmap_sem);
	ret=get_user_pages(current, current->mm, (unsigned long)buf, nr_pages, 1, 0, pages, NULL);
	up_read(¤t->mm->mmap_sem);
	if(ret<0) {
		printk("get_user_pages failed ! %d ",nr_pages);
		return ERR_PARAM;  //ERR_PARAM= -3
	}
	printk("g_sdev[fd] = 0X%p \n",g_sdev[fd]);
	scmd=scsi_get_command(g_sdev[fd],GFP_ATOMIC);
	if(unlikely(!scmd)){
		printk("scmd is NULL ! \n");
		return ERR_BUSY;  // ERR_BUSY=-2
	}
   printk("g_sdev[fd]= 0x%p , scmd is 0X%p ,scmd->device=0x%p ! \n",g_sdev[fd],scmd,scmd->device);
   scmd->cmnd=cdb;  
   scmd->cmd_len=cdb_len;
   scmd->sdb.length=len;  //
   scmd->sdb.resid=0;   //do not know how to use !
   scmd->sdb.table.nents=nr_pages;
   scmd->sdb.table.orig_nents=nr_pages;
  
   scmd->sdb.table.sgl=sg;
   for(nr=0;nr<nr_pages-1;nr++)
   {
	  scmd->sdb.table.sgl[nr].length=PAGE_SIZE;
	  scmd->sdb.table.sgl[nr].page_link = (unsigned long )pages[nr];    //page buf
	  scmd->sdb.table.sgl[nr].offset=0;            //buf offset
	  scmd->sdb.table.sgl[nr].dma_address = page_to_phys(pages[nr]);
	  scmd->sdb.table.sgl[nr].dma_length = PAGE_SIZE;
   }
	scmd->sdb.table.sgl[nr].length=((len%PAGE_SIZE)?(len%PAGE_SIZE):PAGE_SIZE);
	scmd->sdb.table.sgl[nr].page_link = (unsigned long )pages[nr];    
	scmd->sdb.table.sgl[nr].offset=0;        
	scmd->sdb.table.sgl[nr].dma_address = page_to_phys(pages[nr]);
	scmd->sdb.table.sgl[nr].dma_length = ((len%PAGE_SIZE)?(len%PAGE_SIZE):PAGE_SIZE);;
	sg_mark_end(&(scmd->sdb.table.sgl[nr]));
	if(cmd==DISK_READ) {
		scmd->sc_data_direction= DMA_FROM_DEVICE ; //DMA_TO_DEVICE
	} else if(cmd==DISK_WRITE){ 
		scmd->sc_data_direction= DMA_TO_DEVICE ;
	}         
	scmd->transfersize= 512;   //sector 
	scmd->request = (struct request*)res;
	printk("start to queuecommand ! \n");
   
	/*char *buffer=(char *)kmalloc(1024,GFP_KERNEL);
	struct scsi_sense_hdr sshdr;
	ret = scsi_execute_req(g_sdev[fd],cdb,scmd->sc_data_direction,buffer,512,&sshdr,6*HZ,3,NULL);
	printk("scsi_execute ret= %d  read buffer= %s\n",ret,buffer);
	printk("sshdr {0x%x, 0x%x, %x, %x, %x, %x, %x, %x} !\n",sshdr.response_code,sshdr.sense_key,sshdr.asc,sshdr.ascq,sshdr.byte4,sshdr.byte4,sshdr.byte5,sshdr.byte6,sshdr.additional_length);
   */
	//if(!(g_sdev[fd]->host)){printk("g_sdev[fd]->host \n"); return -1;}
	// if(!(g_sdev[fd]->host->hostt)){printk("g_sdev[fd]->host->hostt \n"); return -1;}
	// if(!(g_sdev[fd]->host->hostt->queuecommand)){printk("g_sdev[fd]->host->queuecommand \n"); return -1;}
    if(g_sdev[fd]->sdev_state !=SDEV_RUNNING){
		printk("this scsi device 0x%p not running !",g_sdev[fd]);
		return ERR_BUSY;   //ERR_BUSY=-2
    }
	spin_lock_irqsave(scmd->device->host->host_lock,flags);
	printk("spink lock !\n"); 
	ret=scmd->device->host->hostt->queuecommand(scmd,diskio_done);
        //ret=g_sdev[fd]->host->hostt->queuecommand(scmd, diskio_done);
	spin_unlock_irqrestore(scmd->device->host->host_lock,flags); 
    printk("queuecommand return %x --%d \n",ret,ret);  
	
	return  ret;  //will be change
}

static int do_sync(int cmd,int fd,uint64 seq_num){ 
	 unsigned char *cdb = NULL;
	 int ret=0;
	 int i=0;
	 unsigned long flags;
	 struct scsi_cmnd *scmd;
	 struct page ** pages; 
	 struct scatterlist *sg;
	 struct cmd_result *res= get_one_node(free_queue_head, free_list);
	 if (!res) return ERR_BUSY; //ERR_BUSY=-2
	 pages = res->pages;
	 sg=res->sgl;
	 cdb = res->cdb;
	 res->seq_num = seq_num;
	 if(cmd!=DISK_SYNC){
		  printk("do_sync error !\n");
		  return ERR_PARAM;  //ERR_PARAM=-3
	 } 
	 cdb[0]=0x35;
	 for(i=1;i<10;i++){
	  cdb[i]=0;
	 }
	printk("cmd= %d \n",cmd);
	for(i=0;i<10;i++)
	printk("cdb[%d]= 0x%x ==%d !\n",i,cdb[i],cdb[i]);
	scmd=scsi_get_command(g_sdev[fd],GFP_ATOMIC);
	if(unlikely(!scmd)){
		printk("scmd is NULL ! \n");
		return ERR_NO_DEVICE;  //ERR_NO_DEVICE = -1
	} 
   printk("g_sdev[fd]= 0x%p , scmd is 0X%p ,scmd->device=0x%p ! \n",g_sdev[fd],scmd,scmd->device);
   scmd->cmnd=cdb;  
   scmd->cmd_len=10;
   //scmd->sdb=NULL;
   /*scmd->sdb.length=0;  //
   scmd->sdb.resid=0;   //do not know how to use !
   scmd->sdb.table.nents=0;
   scmd->sdb.table.orig_nents=0;
   scmd->sdb.table.sgl=NULL;
   scmd->sc_data_direction= DMA_TO_DEVICE ;
   */
    scmd->request = (struct request*)res;
      // if(!(g_sdev[fd]->host)){printk("g_sdev[fd]->host \n"); return -1;}
     //  if(!(g_sdev[fd]->host->hostt)){printk("g_sdev[fd]->host->hostt \n"); return -1;}
    // if(!(g_sdev[fd]->host->hostt->queuecommand)){printk("g_sdev[fd]->host->queuecommand \n"); return -1;}
    if(g_sdev[fd]->sdev_state !=SDEV_RUNNING){
	   printk("this scsi device 0x%p not running !",g_sdev[fd]);
	   return ERR_BUSY;   //ERR_BUSY=-2
    }
   spin_lock_irqsave(scmd->device->host->host_lock,flags);
	printk("spink lock !\n"); 
	ret=scmd->device->host->hostt->queuecommand(scmd,diskio_done);
        //ret=g_sdev[fd]->host->hostt->queuecommand(scmd, diskio_done);
	spin_unlock_irqrestore(scmd->device->host->host_lock,flags);
    printk("queuecommand return %x --%d \n",ret,ret);
	return  ret;  //will be change
 }
 
 static int userio_done(unsigned char* buf, int cnt)
 {
	 struct user_result {
		  int result;
		  uint64 seq_num;
		  struct scsi_sense_hdr sshr;
	 };
	struct cmd_result *result; 
	int i,ret, count = 0;
	int MAX_NUM = cnt < MAX_DONEQUEUE_COUNT? cnt: MAX_DONEQUEUE_COUNT;
      MAX_NUM= MAX_NUM <done_queue_head.count? MAX_NUM:done_queue_head.count;
	wait_for_completion(&com);
	for(i = 0; i < MAX_NUM; i++) {
	   result = get_one_node(done_queue_head, done_list);
	   if(result<=0) {
			printk("userio done get node error ! \n");
			continue;
	  }
	count++;
	ret = copy_to_user((unsigned char*)(buf + i * sizeof(struct user_result)), result, sizeof(struct user_result));
	printk("write to user space result result %d, seq_num %lld\n", result->result, result->seq_num);
	memset(result, 0, sizeof(struct cmd_result));
	add_one_node(result, free_queue_head, free_list);
	if (ret < 0)
		return ERR_PARAM; //ERR_PARAM=-3
	}
	return count;
 }
long diskio_ioctl (struct file *filp,unsigned int cmd, unsigned long arg)
{
	 int fd; 
	 int res=0;
	 int ret_command=-1;
	 struct deviceid devid;
	 struct iocmd_head cmd_head;
	 struct {
	  unsigned char* buf;
	  int count;
	 } user_buf;
    printk(" diskio_ioctl cmd= %d \n",cmd);
	switch(cmd) {
	 case DISK_OPEN:     //cmd=0  open to scsi device
			res = copy_from_user((void *)&devid, (void __user*)arg, sizeof(devid));
		   if(!res){   //res =0 copy_from_user success
			   printk("kernel devid struct { %d,%d,%d ,%d} \n",devid.hostnum,devid.channel,devid.id,devid.lun);
			   fd = io_open(&devid);
			   devid.fd=fd;
			   res = copy_to_user((void __user*)arg, (void *)&devid, sizeof(devid));
			   return fd;  // return index >=0   
		   }
			break;
	 case DISK_READ:    //cmd=1001 read scsi device that in  g_sdev[fd]
		   res=copy_from_user((void *)&cmd_head, (void __user*)arg, sizeof(cmd_head));
		   if(!res){
				printk("kernel read cmd_head struct { %d, 0x%p,%d, %lld } \n",cmd_head.fd,cmd_head.buf,cmd_head.len,cmd_head.offset);
				ret_command=do_io( DISK_READ, cmd_head.fd, cmd_head.buf, cmd_head.len, cmd_head.offset, cmd_head.seq_num);
			}
		   break;
	 case DISK_WRITE:   //cmd=1002  write scsi device that in g_sdev[fd]
		   res=copy_from_user((void *)&cmd_head, (void __user*)arg, sizeof(cmd_head));
		   if(!res){
				printk("kernel write cmd_head struct { %d, 0x%p,%d, %lld } \n",cmd_head.fd,cmd_head.buf,cmd_head.len,cmd_head.offset);
				ret_command=do_io( DISK_WRITE, cmd_head.fd, cmd_head.buf, cmd_head.len, cmd_head.offset, cmd_head.seq_num);
			}
		   break;
	 case DISK_SYNC:
		   res=copy_from_user((void *)&cmd_head, (void __user*)arg, sizeof(cmd_head));
		   if(!res){
				printk("kernel synchronize cache cmd_head struct { %d, 0x%p,%d, %lld } \n",cmd_head.fd,cmd_head.buf,cmd_head.len,cmd_head.offset);
				ret_command=do_sync( DISK_SYNC, cmd_head.fd, cmd_head.seq_num);
			}
		   break;
    case DISK_DONE:  //cmd=1003 callback by scsi_done ,return to user
		   res =copy_from_user(&user_buf, (void __user*)arg, sizeof(user_buf));
			if(!res){
				ret_command = userio_done(user_buf.buf, user_buf.count);
			}
			break;
	 default:  /* redundant, as cmd was checked against MAXNR */
		  printk("ioctl cmd=%d not recognize ! ",cmd);
		  ret_command = ERR_PARAM;  //ERR_PARAM=-3
	}
	return ret_command;   //  =queuecommand return or done count
}
static int diskio_open(struct inode *inode, struct file *file)
{
   /*spin_lock(&spin);
	printk("get spinlock\n");
   if(chardev_count){
		spin_unlock(&spin);
		return -EBUSY;
    }
    chardev_count++;
    spin_unlock(&spin);
 */
    return 0;
}
static int diskio_release(struct inode *inode, struct file *file)
{
   // chardev_count--;
    return 0;
}
   
static const struct file_operations diskio_fops = {
    .owner  = THIS_MODULE,
    .open  = diskio_open,
    //.ioctl = diskio_ioctl,
	.unlocked_ioctl=diskio_ioctl,
	.release = diskio_release
};

static int __init diskio_init(void)
{
    dev_t dev_id;
    int retval;
    
	if (device_major) {
	  dev_id = MKDEV(device_major, 0);
	  retval = register_chrdev_region(dev_id, 1, NAME);
    } else {
	  retval = alloc_chrdev_region(&dev_id, 0, 1, NAME);
	  device_major = MAJOR(dev_id);
    }
    if(retval){
	  printk("diskio chrdev region failed\n");
	  retval = -EBUSY;
	  goto err_chrdev_region;
    }
    
	cdev_init(&diskio_cdev, &diskio_fops);
    retval = cdev_add(&diskio_cdev, dev_id, 1);
    if (retval) {
	  printk("diskio cdev add failed\n");
	  retval = -EBUSY;
	  goto err_cdev_add;
    }
    
	diskio_class = class_create(THIS_MODULE, NAME);
    if (IS_ERR(diskio_class)) {
	  printk("%s: class_create failed\n", NAME);
	  retval = PTR_ERR(diskio_class);
	  goto err_class_create;
    }
 
	diskio_device = device_create(diskio_class, NULL, dev_id,  NULL, "%s", NAME);
    if (IS_ERR(diskio_device)) {
	  printk("%s: class_device_create failed\n", NAME);
	  retval = PTR_ERR(diskio_device);
	  goto err_device_create;
    }
	printk("cdev:board_version:%d\n",chardev_var);
    init_MUTEX(&sem);
    spin_lock_init(&spin);
	init_queue_head(&free_queue_head);
	init_queue_head(&done_queue_head);
	init_free_queue();
    return 0;
err_device_create:
    class_destroy(diskio_class);
err_class_create:
    cdev_del(&diskio_cdev);
err_cdev_add:
    unregister_chrdev_region(dev_id, 1);
err_chrdev_region:
    return retval;
}
static void __exit diskio_exit(void)
{
    dev_t dev_id = MKDEV(device_major, 0);
    device_destroy(diskio_class, dev_id);
    class_destroy(diskio_class);
    cdev_del(&diskio_cdev);
    unregister_chrdev_region(dev_id, 1);
	del_queue(free_queue_head, free_list);
	del_queue(done_queue_head, done_list);
}
module_init(diskio_init);
module_exit(diskio_exit);
MODULE_DESCRIPTION("disk io test Driver");
MODULE_LICENSE("GPL");

基于BIO  读写硬盘-----------裸盘读写

/*
 This block device access with bio.
*/
#include <linux/fs.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/cdev.h>
#include <linux/device.h>
#include <asm/uaccess.h>
#include <linux/semaphore.h>
#include <linux/spinlock.h>
#include <linux/delay.h>
#include <linux/scatterlist.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <scsi/scsi.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_transport.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_dbg.h>
#include <scsi/scsi_eh.h>
#include <linux/list.h>
#define NAME "disk_test_io"
#define DISK_OPEN 1000
#define DISK_READ 1001
#define DISK_WRITE 1002
#define DISK_DONE 1003
#define DISK_SYNC  1004
#define MAX_FREQUEUE_COUNT 1024
#define MAX_DONEQUEUE_COUNT 16
typedef unsigned int uint32; 
typedef unsigned long uint64; 
typedef unsigned char uint8;
static int chardev_var = 0;
//static int chardev_count = 0;
static struct semaphore sem;
static spinlock_t spin;
static struct cdev diskio_cdev;
static struct class *diskio_class;
static struct device *diskio_device;
static int device_major = 0;

struct block_device *g_bdev[1024]={NULL};
struct deviceid {
	int major;
	int minor;
	int fd;
};

struct iocmd_head{
	int fd;
	char *buf;
	uint32 len;     //  x*4k
	uint32 offset;  //  x*512   sector
	uint64 seq_num; 
};
struct cmd_result{
	int result;
	uint64 seq_num;
	struct page* pages[128];
	struct list_head free_list;
	struct list_head done_list;
};
struct queue_head{
	spinlock_t lock; 
	struct list_head list;
	int count;
};
struct queue_head free_queue_head;
struct queue_head done_queue_head;
DECLARE_COMPLETION_ONSTACK(com);
#define del_queue(head, member) ({   \
    struct cmd_result *pos,*n; \
    if(!list_empty(&head.list)){  \
    spin_lock(&head.lock); \
    list_for_each_entry_safe(pos, n, &head.list, member)        \
    kfree(pos); \
    head.count=0;\
    spin_unlock(&head.lock); \
    } \
   })
#define get_one_node(head, member) ({  \
         struct cmd_result *p = NULL;\
         spin_lock(&(head).lock);\
         if(list_empty(&head.list)) {\
          spin_unlock(&head.lock);\
          p = NULL;\
         }\
         else {\
         p = container_of(head.list.next, struct cmd_result, member);\
         list_del(head.list.next);\
         (head).count--;\
         spin_unlock(&(head).lock);}\
         p;\
    })  
         
#define add_one_node(new, head, member) ({ \
     spin_lock(&head.lock);\
     head.count++;\
     list_add(&(new)->member,&head.list);\
     spin_unlock(&head.lock);\
    })         
void init_queue_head(struct queue_head * q)
{
	INIT_LIST_HEAD(&q->list);
	q->count = 0;
	spin_lock_init(&q->lock);
}
void init_free_queue(void)
{
	int i;
	struct cmd_result *res = NULL;
	for(i = 0; i < MAX_FREQUEUE_COUNT; i++) {
		res = kzalloc(sizeof(struct cmd_result), GFP_KERNEL);
		if(!res){printk("kzalloc error ! \n");}
		add_one_node(res, free_queue_head, free_list);
	}
}

int io_open(struct deviceid *dev_id){
    int index=0;
	dev_t dev=MKDEV(dev_id->major,dev_id->minor);
	struct block_device *bdev=open_by_devnum(dev,FMODE_READ|FMODE_WRITE);
	if(IS_ERR(bdev)) {
		printk("error:failed to open block device [ %d :%d ] !",dev_id->major,dev_id->minor);
		return PTR_ERR(bdev);
	}
	for(index=0;index<1024;index++){                //this bdev already exists  
		if(g_bdev[index]&&(g_bdev[index]==bdev))
			return index;
	}
	for(index=0;index<1024;index++){               //this bdev is new ,so add to g_sdev
		if(g_bdev[index]==NULL){
			g_bdev[index]=bdev;
			printk("the new bdev=0x%p,\n",bdev);
			return index;
		}
	}
}
void bdev_done(struct bio*bio,int err ){
	struct cmd_result *res = (struct cmd_result*)bio->bi_private;
	printk("bdev_done !\n");
	if (res == NULL)
		goto out;
	printk("bdev done seq num  %ld\n", res->seq_num);
    res->result=err;
	add_one_node(res, done_queue_head, done_list);
	printk("execute generic_make_request return result=%x  \n",res->result);
	printk("frequeue count = %d,  done queue count == %d\n", free_queue_head.count, done_queue_head.count);
out:
   complete(&com);
}
static int do_io(int cmd , int fd,  char * buf, uint32 len, uint32 offset, uint64 seq_num)
{      
	int ret=0,nr=0,err=0;
	struct bio *bio=NULL;
	struct page ** pages=NULL;
    int nr_pages=((uint64)buf+len+PAGE_SIZE-1)/PAGE_SIZE-(uint64)buf/PAGE_SIZE;
	struct cmd_result *res = get_one_node(free_queue_head, free_list);
	if (!res) return -2;
	pages = res->pages;   //tmp
	down_read(¤t->mm->mmap_sem);
	ret=get_user_pages(current, current->mm, (unsigned long)buf, nr_pages, 1, 0, pages, NULL);
	up_read(¤t->mm->mmap_sem);
	if(ret<0) {
		printk("get_user_pages failed ! %d ",nr_pages);
		return ret;
	}
	bio=bio_alloc(GFP_KERNEL,nr_pages);
	bio->bi_bdev=g_bdev[fd]; 
	//bio->bi_flags=
	if(cmd==DISK_READ){
		bio->bi_rw |=0;
	}else if(cmd==DISK_WRITE){
		bio->bi_rw |= 1;
	}else {
		printk("do_io fail ! cmd=%d ",cmd);
		return -1;
	}
	bio->bi_sector=offset;    //start sector
	bio->bi_size=0;   // residual io count
	bio->bi_end_io=bdev_done;   // bio done callback func
	bio->bi_private=res;
    for(nr=0; nr<nr_pages-1; nr++)
	{
	   err=bio_add_page(bio,pages[nr],PAGE_SIZE,0);
	   if(err<=0) {
			printk("bio add page fail !");
			return err;
	   }
	}
    err=bio_add_page(bio,pages[nr], ((len%PAGE_SIZE)?(len%PAGE_SIZE):PAGE_SIZE), 0);
    if(err<=0){
	  printk("end page add to bio fail !");
	  return err;
    }
    generic_make_request(bio);
    return 0;
 }
 static int userio_done(unsigned char* buf, int cnt)
{
	int MAX_NUM;
	struct cmd_result *result;
	MAX_NUM= cnt < MAX_DONEQUEUE_COUNT? cnt: MAX_DONEQUEUE_COUNT;
    MAX_NUM= MAX_NUM <done_queue_head.count? MAX_NUM:done_queue_head.count;
	int i,ret, count = 0;
	struct user_result {
		int result;
		uint64 seq_num;
		//struct scsi_sense_hdr sshr;
	};
	wait_for_completion(&com);
	for(i = 0; i < MAX_NUM; i++) {
		result = get_one_node(done_queue_head, done_list);
		if(result<=0) {
			printk("userio done get node error ! \n");
			continue;
		}
		count++;
		ret = copy_to_user((unsigned char*)(buf + i * sizeof(struct user_result)), result, sizeof(struct user_result));
		printk("write to user space result result %d, seq_num %ld\n", result->result, result->seq_num);
		memset(result, 0, sizeof(struct cmd_result));
		add_one_node(result, free_queue_head, free_list);
		if (ret < 0)
			return -1;
	}
	return count;
}

int cdev_ioctl (struct inode *inode, struct file *filp,uint32 cmd, uint64 arg)
{
	int res=0;
	int ret_command=-1;
	struct deviceid devid;
	struct iocmd_head cmd_head;
	struct {
	  unsigned char* buf;
	  int count;
	} user_buf;
    printk(" cdev_ioctl cmd= %d \n",cmd);
	switch(cmd) {
		case DISK_OPEN:     //cmd=0  open to scsi device
			res = copy_from_user((void *)&devid, (void __user*)arg, sizeof(devid));
		   if(!res){   //res =0 copy_from_user success
				printk("kernel devid struct { %d,%d} \n",devid.major,devid.minor);
				ret_command = io_open(&devid);
				if(ret_command >=0){
					devid.fd=ret_command;
					res = copy_to_user((void __user*)arg, (void *)&devid, sizeof(devid));
					printk("== copy_to_user ======res= %d \n",res);
					return res;  // 没有拷贝的完的字节数
				}
		   }
		  break;
		case DISK_READ:    //cmd=1001 read scsi device that in  g_sdev[fd]
			res=copy_from_user((void *)&cmd_head, (void __user*)arg, sizeof(cmd_head));
			if(!res){
				printk("kernel read cmd_head struct { %d, 0x%p,%d, %d } \n",cmd_head.fd,cmd_head.buf,cmd_head.len,cmd_head.offset);
				ret_command=do_io( DISK_READ, cmd_head.fd, cmd_head.buf, cmd_head.len, cmd_head.offset, cmd_head.seq_num);
			}
		   break;
		case DISK_WRITE:   //cmd=1002  write scsi device that in g_sdev[fd]
		   res=copy_from_user((void *)&cmd_head, (void __user*)arg, sizeof(cmd_head));
		   if(!res){
				printk("kernel write cmd_head struct { %d, 0x%p,%d, %d } \n",cmd_head.fd,cmd_head.buf,cmd_head.len,cmd_head.offset);
				ret_command=do_io( DISK_WRITE, cmd_head.fd, cmd_head.buf, cmd_head.len, cmd_head.offset, cmd_head.seq_num);
			}
		   break;
		/*case DISK_SYNC:
		   res=copy_from_user((void *)&cmd_head, (void __user*)arg, sizeof(cmd_head));
		   if(!res){
				printk("kernel synchronize cache cmd_head struct { %d, 0x%p,%d, %d } \n",cmd_head.fd,cmd_head.buf,cmd_head.len,cmd_head.offset);
				ret_command=do_sync( DISK_SYNC, cmd_head.fd, cmd_head.seq_num);
			}
		   break;
		 */ 
		case DISK_DONE:  //cmd=1003 callback by scsi_done ,return to user
			res =copy_from_user(&user_buf, (void __user*)arg, sizeof(user_buf));
			if(!res){
				ret_command = userio_done(user_buf.buf, user_buf.count);
			}
			break;
		default:  /* redundant, as cmd was checked against MAXNR */
		  printk("ioctl cmd=%d not recognize ! ",cmd);
		  ret_command = -3;
	}
	return ret_command;
}
static int cdev_open(struct inode *inode, struct file *file)
{
    return 0;
}
static int cdev_release(struct inode *inode, struct file *file)
{
    return 0;
}
static const struct file_operations diskio_fops = {
    .owner  = THIS_MODULE,
    .open  = cdev_open,
    .ioctl = cdev_ioctl,
	.release = cdev_release
};
static int __init diskio_init(void)
{
    dev_t dev_id;
    int retval;
    
	if (device_major) {
		dev_id = MKDEV(device_major, 0);
		retval = register_chrdev_region(dev_id, 1, NAME);
    } else {
		retval = alloc_chrdev_region(&dev_id, 0, 1, NAME);
		device_major = MAJOR(dev_id);
    }
    if(retval){
		printk("diskio chrdev region failed\n");
		retval = -EBUSY;
		goto err_chrdev_region;
    }
	cdev_init(&diskio_cdev, &diskio_fops);
    retval = cdev_add(&diskio_cdev, dev_id, 1);
    if (retval) {
		printk("diskio cdev add failed\n");
		retval = -EBUSY;
		goto err_cdev_add;
    }
	diskio_class = class_create(THIS_MODULE, NAME);
    if (IS_ERR(diskio_class)) {
		printk("%s: class_create failed\n", NAME);
		retval = PTR_ERR(diskio_class);
		goto err_class_create;
    }
	diskio_device = device_create(diskio_class, NULL, dev_id,  NULL, "%s", NAME);
    if (IS_ERR(diskio_device)) {
		printk("%s: class_device_create failed\n", NAME);
		retval = PTR_ERR(diskio_device);
		goto err_device_create;
    }
    printk("cdev:board_version:%d\n",chardev_var);
    init_MUTEX(&sem);
    spin_lock_init(&spin);
	init_queue_head(&free_queue_head);
	init_queue_head(&done_queue_head);
	init_free_queue();
    return 0;
err_device_create:
    class_destroy(diskio_class);
err_class_create:
    cdev_del(&diskio_cdev);
err_cdev_add:
    unregister_chrdev_region(dev_id, 1);
err_chrdev_region:
    return retval;
}
static void __exit diskio_exit(void)
{
    dev_t dev_id = MKDEV(device_major, 0);
    device_destroy(diskio_class, dev_id);
    class_destroy(diskio_class);
    cdev_del(&diskio_cdev);
    unregister_chrdev_region(dev_id, 1);
	del_queue(free_queue_head, free_list);
	del_queue(done_queue_head, done_list);
}
module_init(diskio_init);
module_exit(diskio_exit);
MODULE_DESCRIPTION("disk io test Driver");
MODULE_LICENSE("GPL");

用户态测试程序：(接口基本一样)

#include <stdio.h>
#include <stdlib.h>
#include <malloc.h>
#include <unistd.h>
#define PAGE_SIZE (1<<12)
#define DISK_OPEN 1000
#define DISK_READ 1001
#define DISK_WRITE 1002
#define DISK_DONE 1003
#define DISK_SYNC  1004
typedef unsigned int uint32; 
typedef unsigned long uint64; 
typedef unsigned char uint8;
struct deviceid {
    uint32 hostnum;
	uint32 channel;
	uint32 id;
	uint32 lun;
	int fd;
};
struct iocmd_head{
    uint32 fd; //sdev[index]  --> index = fd
	char *buf;
	uint32 len;   // len=n*512
	uint32 offset;  //offset is sector num
	uint64 seq_num; 
};
struct user_result{
  int result;
  uint64 seq_num;
  uint8 response_code; 
  uint8 sense_key;
  uint8 asc;
  uint8 ascq;
  uint8 byte4;
  uint8 byte5;
  uint8 byte6;
  uint8 additional_length; 
};

struct done_input{
  struct user_result * buffer;  // struct user_result* 
  int count;
 };
 
int main(int argc,char* argv[])
{
	unsigned char *array = valloc(1024);
	unsigned char *read_arr = valloc(1024);
	int ret,ret_num;
	int i=0;
	for(i=0;i<1024;i++){
		array[i]='g';
	}
	if(argc!=5){
		printf("run this program as ./a.out hostnum channel id lun !\n"); 
		return 0;
	}
	struct deviceid mydev;
	mydev.hostnum=*argv[1]-'0';
	mydev.channel=*argv[2]-'0';
	mydev.id=*argv[3]-'0';
	mydev.lun=*argv[4]-'0';
	struct iocmd_head iocmd={ 0 };
	iocmd.buf=array;
	iocmd.len=512;
	iocmd.offset=0;            //LBA  %512
    iocmd.seq_num=1;
 
	printf("user pass devid struct {%d, %d, %d,% d} \n", mydev.hostnum, mydev.channel, mydev.id, mydev.lun);
 
	int fd = open("/dev/disk_test_io", 0070);
	 //begin to open
	 int devid = ioctl(fd,DISK_OPEN,&mydev);          //open  one scsi dev
	 if(devid==-1 ) { 
		printf("ioctrl open device error !\n");       //can not find device   max devidid 1023
		return 0;
	 }
	printf("open this dev with index= %d \n", devid);
 
	//begin to write
	iocmd.fd=devid;
	printf("array= 0X%x ,iocmd.buf=0X%x\n",array,iocmd.buf);
	printf("user pass iocmd struct {%d, 0x%x, %d,% d} \n", iocmd.fd, iocmd.buf, iocmd.len, iocmd.offset);
 
	ret=ioctl(fd, DISK_WRITE, &iocmd);
	if(ret!=0){
		printf("ioctl write error return %d --0x%x \n",ret,ret);
		return 0;
	}
	printf("  write IOCTRL succes =====ret =%d array=%s \n",ret,array);
	sleep(5);
	//begin to read
	iocmd.buf=read_arr;
	ret=ioctl(fd, DISK_READ, &iocmd);
	if(ret!=0){
		printf(" 2:ioctl read return %d --0x%x \n",ret,ret);
		return 0;
	}
	sleep(5);
	printf(" 2:read IOCTRL succes=====ret =%d read_arr=%s \n",ret,read_arr);
  //begin ioctrl done
	struct user_result result_buf[20];
	struct done_input don_buf={
	  .buffer=result_buf,
	  .count=10,
	};
  ret_num = ioctl(fd, DISK_DONE, &don_buf);
	if (ret_num) {
		int i;
		printf("get done io %d\n", ret_num);
		for ( i = 0; i < ret_num; i ++)
		printf("result %d, seq_num %ld, response_code %d, sense_key %d, asc %d, ascq %d\n", result_buf[i].result, result_buf[i].seq_num,result_buf[i].response_code, result_buf[i].sense_key,result_buf[i].asc,result_buf[i].ascq);
	}
	ret_num = ioctl(fd, DISK_SYNC, &iocmd);
	if (ret_num!=0) {
          printf("ioctl sync error ret=%d !\n",ret_num); 
		return 0;
	}
	printf("ioctrl sync success !\n");
	ret_num = ioctl(fd, DISK_DONE, &don_buf);
	if (ret_num) {
		int i;
		printf("get done io %d\n", ret_num);
		for ( i = 0; i < ret_num; i ++)
			printf("result %d, seq_num %ld, response_code %d, sense_key %d, asc %d, ascq %d\n", result_buf[i].result, result_buf[i].seq_num,result_buf[i].response_code, result_buf[i].sense_key,result_buf[i].asc,result_buf[i].ascq);
	}
}