先讲讲本次主题的项目需求:
1、透传数据,若无连接则必须缓存数据(数据上传的环境很不理想,很可能1周后才有人处理)。
2、最差情况:缓存百兆字节数据;
2、缓存的数据按先进先出顺序;
3、缓存数据接收每次100~4K字节,每段数据总长度10K~1M不等。
4、stm32平台64K内存,64KROM;(由于USB Host驱动,再加上系统及任务本身耗费,内存裕量约40k);
设计方案:
首先,开发版采用的是NOR flash, 其擦写寿命约为10w次。如果采用fat文件系统的话,因为每次追加数据很少,那么势必频繁更新目录项与fat表。那么flash上这块区域根本坚持不了多久(粗略估计1年)就坏了。并且fat文件系统是非常古老的系统,抗掉电,各种健壮性都很差。所幸早已有专用于flash的文件系统如JFFS2。
但是JFFS2也有其缺点:1、发现该文件系统占用的内存随flash的大小增大,如果是256Mflash,需要的内存已经超出了本次项目需求。2、文件系统加载时间也是随flash的大小增大(达到几分钟级别),所以不是很适用本次项目。
因而本项目决定采用了ringbuffer方式,根据flash的擦除特性,并吸收JFFS2的一些优点来完善掉电恢复,写平衡,擦除平衡。延长寿命。
jffs2的mask方法,不同的文件类型不同的type。同样采用结构体进行辨别控制:
struct fs_dir{
uint16_t magic;
uint16_t type;
uint16_t ino;
......
uint16 hcrc;
};
讲解:
为了达到快速遍历重新恢复上次掉电前状态,同时为了免除删除数据和不删除数据在同一个擦除块,而进行搬运数据。缓存数据以擦除块大小(假设4k)为单位进行保存。每个缓存数据块(第一页page,保存目录项等数据,称为目录页)。这样只需对4k字节对齐地址进行扫描即可重建快速定位了。为了更快的进行加载,目录页中,可在添加文件小结数据,指出其大小,占用空间等。同理文件的删除亦可以放在目录页(删除存在的原因是为了下次加载时,识别出这是无效数据)。
另外为了防止擦除到一半意外停止了,也学习JFFS2,在擦除完成后加入擦除Mask。后续检查Mask即可知道是否已擦除,在检查后续两字符可知道这个块是否使用过(这里假设所有flash写入都是自己写入,都是顺序写入)。当然在数据项中包含着数据的CRC校验,以确保这个数据正常工作。
代码构思调试总共花了二周,略显粗糙,不过够用就行。话不多说上关键代码。
/*
* easyfs's ringbuffer at flash
*
* ebtail -> ---------------------------------------- <---- erase block 0xXX0000 / page 0
* | EraseMask| Dir | DirSum | DirDel |
* | |
* --------------------------------------- <---- page 1 0xXXX100
* | Node | Data ... |
* | Data ... |
*
* ..................................
*
* | | <---- erase block 0xXX0FFF
* ------------------------------------- <---- erase block 0xXX1000 / page 16
* | EraseMask| Node | Data ... |
*
* ..................................
*
* ebhdr ---> -----------------------------------
* | EraseMask| 0xFF 0xFF ....... | <---- erase block 0xXX2000 / page 16
*
*/
int isHdrType(uint8_t type, void* buf, uint32_t maxsize)
{
Efs_PageUnknow_t *hdr;
uint32_t crcofs;
uint32_t hcrc;
uint32_t crc;
hdr = (Efs_PageUnknow_t*)buf;
if(hdr->magic != EFS_PAGE_MAGIC){
//LOG(LOG_INFO, "no HDR magic: %02x %02x\n", ((char*)buf)[0], ((char*)buf)[1]);
return 0;
}
if(type != hdr->type){
if(!(type == 2 && hdr->type == 1))
LOG(LOG_INFO, "actualy type %d(except %d)\n", hdr->type, type);
return 0;
}
switch(hdr->type){
case EFS_TYPE_ERASE:
crcofs = offsetof(Efs_PageEraseHeader_t, hcrc);
hcrc = ((Efs_PageEraseHeader_t*)buf)->hcrc;
break;
case EFS_TYPE_NODE:
crcofs = offsetof(Efs_PageNodeHeader_t, hcrc);
hcrc = ((Efs_PageNodeHeader_t*)buf)->hcrc;
break;
case EFS_TYPE_DIR:
crcofs = offsetof(Efs_PageDirHeader_t, hcrc);
hcrc = ((Efs_PageDirHeader_t*)buf)->hcrc;
break;
case EFS_TYPE_DIRSUM:
crcofs = offsetof(Efs_PageDirSumHeader_t, hcrc);
hcrc = ((Efs_PageDirSumHeader_t*)buf)->hcrc;
break;
case EFS_TYPE_DIRDEL:
crcofs = offsetof(Efs_PageDirDelHeade_t, hcrc);
hcrc = ((Efs_PageDirDelHeade_t*)buf)->hcrc;
break;
default:
LOG(LOG_WARN, "efs unknow type\n");
return 0;
}
if(crcofs > maxsize){
LOG(LOG_WARN, "efs hdr crcofs large than maxsize %d\n", maxsize);
return 0;
}
crc = crc32(0, buf, crcofs);
if(crc == hcrc){
return 1;
}
else{
LOG(LOG_WARN, "efs hdr crc error: type %d crcofs %d, crc %x, hcrc %x\n", hdr->type, crcofs, crc, hcrc);
DUMP("CRC CHECK", buf, crcofs+4);
return 0;
}
}
void fileStreamIn(File_Opt opt, uint8_t* buf, uint32_t size)
{
static Efs_PageDirHeader_t dh;
static Efs_PageDirSumHeader_t ds;
static Efs_PageNodeHeader_t n;
static Efs_PageDirDelHeade_t dl;
dh.magic = EFS_PAGE_MAGIC;
dh.type = EFS_TYPE_DIR;
ds.magic = EFS_PAGE_MAGIC;
ds.type = EFS_TYPE_DIRSUM;
n.magic = EFS_PAGE_MAGIC;
n.type = EFS_TYPE_NODE;
dl.magic = EFS_PAGE_MAGIC;
dl.type = EFS_TYPE_DIRDEL;
static uint32_t fileEbofs = 0;
uint8_t* data;
uint32_t ctlen;
uint32_t valid_left;
uint32_t e_offset;
srand(HAL_GetTick());
switch(opt){
case File_PUSH:
dh.id = rand();
dh.time = highesttime + HAL_GetTick();
dh.hcrc = crc32(0, (uint8_t*)&dh, offsetof(Efs_PageDirHeader_t, hcrc));
fileEbofs = ebhdr;
//write
writeDir(&dh, ebhdr + sizeof(Efs_PageEraseHeader_t));
//syn
ds.id = dh.id;
ds.tldsize = 0;
ds.ebsize = 0;
n.id = dh.id;
fileEbofs = ebhdr;
addEbHdr(PAGE_SIZE);
if(size == 8 && buf){
memcpy(buf, &dh.id, 4);
memcpy(buf+4, &dh.time, 4);
}
break;
case File_WRITE:
data = buf;
while(size){
e_offset = ((ebhdr&0xf00)? 0:sizeof(Efs_PageEraseHeader_t));
valid_left = EARSEBLOCK_SIZE - (ebhdr&0xf00)- e_offset - sizeof(Efs_PageNodeHeader_t);
ctlen = size > valid_left? valid_left: size;
n.offset = ds.tldsize;
n.dsize = ctlen;
n.dcrc = crc32(0, data, ctlen);
n.hcrc = crc32(0, (uint8_t*)&n, offsetof(Efs_PageNodeHeader_t, hcrc));
writeNode(&n, ebhdr + e_offset, data, ctlen);
//syn
addEbHdr(ctlen + sizeof(Efs_PageNodeHeader_t) + e_offset);
size -= ctlen;
data += ctlen;
ds.tldsize += ctlen;
}
break;
case File_END:
//ebhdr -> next start position
roundEbHdr();
ds.ebsize = (ebhdr - fileEbofs);
ds.hcrc = crc32(0, (uint8_t*)&ds, offsetof(Efs_PageDirSumHeader_t, hcrc));
writeDirSum(&ds, fileEbofs + sizeof(Efs_PageEraseHeader_t) + sizeof(Efs_PageDirHeader_t));
writeflush();
//syn
break;
case File_DEL:
dl.id = ds.id;
dl.hcrc = crc32(0, (uint8_t*)&dl, offsetof(Efs_PageDirDelHeade_t, hcrc));
writeDirDel(&dl, fileEbofs + sizeof(Efs_PageEraseHeader_t) + sizeof(Efs_PageDirHeader_t) + sizeof(Efs_PageDirSumHeader_t));
writeflush();
//syn
LOG(LOG_INFO, "push file del: id %x, at fileEbofs 0x%08x\n", dh.id, fileEbofs);
break;
default:
break;
}
}
int popTailFile(uint32_t *address, Efs_PageDirHeader_t *d_dh, Efs_PageDirSumHeader_t *d_ds)
{
uint8_t *buf = (uint8_t*)readBuf;
uint32_t bufofs;
uint32_t ebofs = ebtail;
while(1){
if(ebofs >= ebhdr)//to the end
return -1;
bufofs = 0;
//这段可以封装,自己弄吧
read(ebofs, buf, sizeof(Efs_PageEraseHeader_t) + sizeof(Efs_PageDirHeader_t) + sizeof(Efs_PageDirSumHeader_t) + sizeof(Efs_PageDirDelHeade_t));
if(isHdrType(EFS_TYPE_ERASE, buf, sizeof(Efs_PageEraseHeader_t))){
bufofs += sizeof(Efs_PageEraseHeader_t);
if(isHdrType(EFS_TYPE_DIR, buf + bufofs, sizeof(Efs_PageDirHeader_t))){
bufofs += sizeof(Efs_PageDirHeader_t);
if(isHdrType(EFS_TYPE_DIRSUM, buf + bufofs, sizeof(Efs_PageDirSumHeader_t))){
bufofs += sizeof(Efs_PageDirSumHeader_t);
if(!isHdrType(EFS_TYPE_DIRDEL, buf + bufofs, sizeof(Efs_PageDirDelHeade_t))){
//no delete
*address = ebofs;
memcpy(d_dh, buf + sizeof(Efs_PageEraseHeader_t), sizeof(Efs_PageDirHeader_t));
memcpy(d_ds, buf + sizeof(Efs_PageEraseHeader_t) + sizeof(Efs_PageDirHeader_t), sizeof(Efs_PageDirSumHeader_t));
return 0;
}
}
}
}
ebofs += EARSEBLOCK_SIZE;
}
}
int popFile(uint32_t *address, Efs_PageDirHeader_t *d_dh, Efs_PageDirSumHeader_t *d_ds)
{
uint32_t ebofs = ebhdr;
uint8_t *buf = (uint8_t*)readBuf;
uint32_t bufofs;
ebofs = LROUND(ebhdr, EARSEBLOCK_SIZE);
while(1){
bufofs = 0;
read(ebofs, buf, sizeof(Efs_PageEraseHeader_t) + sizeof(Efs_PageDirHeader_t) + sizeof(Efs_PageDirSumHeader_t) + sizeof(Efs_PageDirDelHeade_t));
if(isHdrType(EFS_TYPE_ERASE, buf, sizeof(Efs_PageEraseHeader_t))){
bufofs += sizeof(Efs_PageEraseHeader_t);
if(isHdrType(EFS_TYPE_DIR, buf + bufofs, sizeof(Efs_PageDirHeader_t))){
bufofs += sizeof(Efs_PageDirHeader_t);
if(isHdrType(EFS_TYPE_DIRSUM, buf + bufofs, sizeof(Efs_PageDirSumHeader_t))){
bufofs += sizeof(Efs_PageDirSumHeader_t);
if(!isHdrType(EFS_TYPE_DIRDEL, buf + bufofs, sizeof(Efs_PageDirDelHeade_t))){
//no delete
*address = ebofs;
memcpy(d_dh, buf + sizeof(Efs_PageEraseHeader_t), sizeof(Efs_PageDirHeader_t));
memcpy(d_ds, buf + sizeof(Efs_PageEraseHeader_t) + sizeof(Efs_PageDirHeader_t), sizeof(Efs_PageDirSumHeader_t));
return 0;
}
}
}
}
if(ebofs <= ebtail)
return -1;
if(ebofs >= EARSEBLOCK_SIZE){
ebofs -= EARSEBLOCK_SIZE;
}
else
return -1;
}
}
int fileStreamOut(File_Opt opt, uint8_t* buf, uint32_t size)
{
static Efs_PageDirHeader_t dh;
static Efs_PageDirSumHeader_t ds;
static Efs_PageNodeHeader_t n;
static Efs_PageDirDelHeade_t dl;
dl.magic = EFS_PAGE_MAGIC;
dl.type = EFS_TYPE_DIRDEL;
static uint32_t fileEbofs = 0;
static uint32_t ebofs = 0;
static uint32_t ofs = 0;
static uint8_t pop = 0;
uint32_t seek_ofs;
uint32_t seek_ebofs;
uint8_t* data;
uint32_t ctlen;
uint32_t left;
uint32_t e_offset;
switch(opt){
case File_POPTAIL:
if(0 > popTailFile(&fileEbofs, &dh, &ds))
return 0;
ebofs = fileEbofs + PAGE_SIZE;
ofs = 0;
pop = 1;
if(size >= 8){
memcpy(buf, &dh.id, 4);
memcpy(buf+4, &dh.time, 4);
}
LOG(LOG_INFO, "poptail file: id = %x, time = %d at fileEbofs 0x%08x, ebsize 0x%08x\n", dh.id, dh.time, fileEbofs, ds.ebsize);
return 1;
case File_POP:
if(0 > popFile(&fileEbofs, &dh, &ds))
return 0;
ebofs = fileEbofs + PAGE_SIZE;
ofs = 0;
pop = 0;
if(size == 8 && buf){
memcpy(buf, &dh.id, 4);
memcpy(buf+4, &dh.time, 4);
}
LOG(LOG_INFO, "pop file: id = %x, time = %d at fileEbofs 0x%08x, ebsize 0x%08x\n", dh.id, dh.time, fileEbofs, ds.ebsize);
return 1;
case File_READ:
//to end
if(ebofs - fileEbofs >= ds.ebsize)
return 0;
data = buf;
left = size;
while(left){
e_offset = (ebofs&0xf00)? 0: sizeof(Efs_PageEraseHeader_t);
read(ebofs + e_offset, (uint8_t*)&n, sizeof(Efs_PageNodeHeader_t));
if(isHdrType(EFS_TYPE_NODE, (uint8_t*)&n, sizeof(Efs_PageNodeHeader_t))){
//LOG(LOG_INFO, "pop file ofs %d, n.offset %d, n.dsize %d\n", ofs, n.offset, n.dsize);
if(ofs >= n.offset && ofs < (n.offset + n.dsize)){
uint32_t nodeDataLeft = n.dsize - (ofs - n.offset);
ctlen = left > nodeDataLeft? nodeDataLeft: left;
read(ebofs + e_offset +sizeof(Efs_PageNodeHeader_t) + (ofs - n.offset), data, ctlen);
//syn
ofs += ctlen;
data += ctlen;
left -= ctlen;
//LOG(LOG_INFO, "pop file read ebofs 0x%08x, ctlen %d, left %d\n", ebofs, ctlen, left);
if(ctlen == nodeDataLeft)
ebofs += ROUND(n.dsize + sizeof(Efs_PageNodeHeader_t) + e_offset, PAGE_SIZE);
}
else{
ebofs += ROUND(n.dsize + sizeof(Efs_PageNodeHeader_t) + e_offset, PAGE_SIZE);
}
}
else{
//LOG(LOG_INFO, "hdrCheck: failed at ebofs 0x%08x\n", ebofs);
ebofs += PAGE_SIZE;
}
if(ebofs - fileEbofs >= ds.ebsize)
break;
}
LOG(LOG_INFO, "pop file read: end ebofs 0x%08x, ofs %d size %d\n", ebofs, ofs, size - left);
return size - left;
case File_SEEK:
memcpy(&seek_ofs, &buf, 4);
if(seek_ofs > ds.tldsize)
seek_ofs = ds.tldsize;
if(seek_ofs < ofs){
seek_ebofs = fileEbofs + PAGE_SIZE;
left = seek_ofs;
}
else{
left = seek_ofs - ofs;
seek_ebofs = ebofs;
}
while(left){
e_offset = (seek_ebofs&0xf00)? 0: sizeof(Efs_PageEraseHeader_t);
read(seek_ebofs + e_offset, (uint8_t*)&n, sizeof(Efs_PageNodeHeader_t));
if(isHdrType(EFS_TYPE_NODE, (uint8_t*)&n, sizeof(Efs_PageNodeHeader_t))){
LOG(LOG_INFO, "seek file ofs %d, n.offset %d, n.dsize %d\n", ofs, n.offset, n.dsize);
if(ofs >= n.offset && ofs < (n.offset + n.dsize)){
ofs = seek_ofs;
ebofs = seek_ebofs;
return ofs;
}
else{
seek_ebofs += ROUND(n.dsize + e_offset + sizeof(Efs_PageNodeHeader_t), PAGE_SIZE);
}
}
else{
//LOG(LOG_INFO, "hdrCheck: failed at ebofs 0x%08x\n", ebofs);
seek_ebofs += PAGE_SIZE;
}
if(seek_ebofs - fileEbofs >= ds.ebsize)
break;
}
break;
case File_DEL:
//ebtail -> next start position
if(pop)
addEbTail(ds.ebsize);
dl.id = ds.id;
dl.hcrc = crc32(0, (uint8_t*)&dl, offsetof(Efs_PageDirDelHeade_t, hcrc));
writeDirDel(&dl, fileEbofs + sizeof(Efs_PageEraseHeader_t) + sizeof(Efs_PageDirHeader_t) + sizeof(Efs_PageDirSumHeader_t));
writeflush();
//syn
LOG(LOG_INFO, "pop file del: id %x, at fileEbofs 0x%08x\n", dh.id, fileEbofs);
break;
default:
break;
}
return 0;
}