mixin代码
import re
from django.db import transaction
class Ignore(Exception):
"""ignore import"""
pass
class ImportMixin(object):
"""
导入Mixin
line_regex:the regex for process lines in file
line_split:the split of line
field_mapper:a list or tuple of mapper rules to convert line split list to the result,
the rule is:
args = line.split(import_split)
1.if the field_mapper[i] is integer bigger than zero,then the model's fields[i] = some_fun(args[field_mapper[i]])
2.if the field_mapper[i] is tuple or list,the model's fields[i] = some_fun(*[args[k] for k in field_mapper[i]])
3.if the field_mapper[i] is None,the fields[i] = field_namedefault()
example:
class A(object):
field1=xxxx
field2=xxxx
field3=xxxx
the file1 line is field2,field1,the mapper is (1,0,None)
the file2 line is field2,field1-part1,field1-part2,the mapper is ((1,2),0,None)
"""
line_regex = r'.*'
line_split = ','
line_pre_processor=()#pre processor before line is import
file_pre_processor=()#pre processor before file is import
file_types=('csv',)
field_mapper=None
is_ignore = False
@transaction.commit_on_success
def import_from_file(self,file):
if not self.is_valid_file(file):
raise Exception,u"the file type is not a valid type"
if self.field_mapper is None:
self.field_mapper = range(0,len(self.model._meta.fields))
if len(self.model._meta.fields)!=len(self.field_mapper):
raise IndexError,u"the field_mapper's length not equal fields count"
num,line,obj=0,None,None
try:
self._pre_process_file(file)
for line in file:
line = self._pre_process_line(line)
if self.line_regex and not re.match(self.line_regex,line):
raise ValueError,"import line must '%s'"%(self.line_regex)
args = line.split(self.line_split)
try:
field_values = dict([self._field_converter(args=args,index=index) for index in range(0,len(self.field_mapper))])
obj = self.model(**field_values)
obj.validate()
obj.save()
except Ignore,e:
if self.is_ignore:#if ignore,continue,else raise exception and stop import
pass
else:
raise e
num=num+1
except Exception,e:
raise Exception,u"failed import from line(%i),the reason is %s,the line content '%s'"%(num,e,line)
finally:
if not file.closed: file.close()
return obj
def _field_converter(self,args=None,index=None):
"""
根据field_mapper规则将args用fieldname_converter转化成结果,
如未定义fieldname_converter,则转化器参数即为返回值,即lambda x:x
如field_mapper[i] 为None,则调用fieldname_default
"""
if args is None or index is None or not isinstance(index,int):
raise ValueError,u'the import args and index can not be None!,and index must be integer'
field = self.model._meta.fields[index]
try:
converter = getattr(self,'%s_converter'%field.name.lower())#the fiedname id realFieldName.lower()
except AttributeError:
converter = lambda x:x
try:
default = getattr(self,'%s_default'%field.name.lower())
except AttributeError:
default = lambda:None
if isinstance(self.field_mapper[index],(list,tuple)):
value = converter(*[args[k] for k in self.field_mapper[index]])
elif isinstance(self.field_mapper[index],int):
value = converter(args[self.field_mapper[index]])
elif self.field_mapper[index] is None:
value = default()
try:
return (field.name,value)
except NameError:
raise ValueError,u'the element(%s) in field_mapper is not instance of list,tuple,int or None'%arg
def _pre_process_line(self,line):
"""
to pre processor line by processor register in line_pre_processor
"""
pre_processor=None
try:
for name in self.line_pre_processor:
pre_processor = getattr(self,'%s_line_pre_processor'%name)
line = pre_processor(line)
return line
except AttributeError,e:
raise AttributeError,u'%s,the pre_processor(%s) is not difine'%(e,pre_processor)
def trim_line_pre_processor(self,line):
"""
trim line's space both left and right
if you want use it,you can configure the class attribute line_pre_processor=('trim'),
you also could define other processor,such as unicode_line_pre_processor and so on
"""
return line.strip()
def replace_xOO_line_pre_processor(self,line):
"""
replace \x00
"""
return line.replace('\x00','')
def unicode_line_pre_processor(self,line):
"""
covert line to unicode,
if you want use it,you can configure the class attribute line_pre_processor=('unicode'),
you also could define other processor,such as trim_line_pre_processor and so on
"""
return u'%s'%line
def _pre_process_file(self,file):
"""
to pre processor file by processor register in file_pre_processor
"""
pre_processor=None
try:
for name in self.file_pre_processor:
pre_processor = getattr(self,'%s_file_pre_processor'%name)
file = pre_processor(file)
return file
except AttributeError,e:
raise AttributeError,u'%s,the pre_processor(%s) is not difine'%(e,pre_processor)
def bom_file_pre_processor(self,file):
"""
process utf coding bom
# UTF-8
BOM_UTF8 = ('\xef','\xbb','\xbf')
# UTF-16, little endian
BOM_LE = BOM_UTF16_LE = ('\xff','\xfe')
# UTF-16, big endian
BOM_BE = BOM_UTF16_BE = ('\xfe','\xff')
# UTF-32, little endian
BOM_UTF32_LE = ('\xff','\xfe','\x00','\x00')
# UTF-32, big endian
BOM_UTF32_BE = ('\x00','\x00','\xfe','\xff')
"""
bom = [file.read(1) for i in range(0,4)]
if bom[0] == '\x00' and bom[1] =='\x00' and bom[2]=='\xfe' and bom[3]=='\xff':
pass#BOM_UTF32_BE
elif bom[0] == '\xef' and bom[1]=='\xbb' and bom[2]=='\xbf':
file.seek(3)#BOM_UTF8
elif bom[0] == '\xfe ' and bom[1] == '\xff':
file.seek(2)#BOM_BE and BOM_UTF16_BE
elif bom[0] == '\xff' and bom[1]=='\xfe':
if bom[2]=='\x00' and bom[3]=='\x00':
pass#BOM_UTF32_LE
else:
file.seek(2)#BOM_LE and BOM_UTF16_LE
else:
file.seek(0)
return file
def is_valid_file(self,file):
name = file.name.split('.')
if name[len(name)-1] in self.file_types:
return True
return False
model代码
#model代码
class Bankcard(models.Model,ModelValidateMixin):
id = models.AutoField(primary_key=True,db_column='fid')
cardNo = models.CharField(u'银行卡号',max_length=19,db_column='fcard_no',unique=True)
name = models.CharField(u'户名',max_length=20,db_column='fname')
cardType = models.ForeignKey(BankcardType,db_column='ftype_id',verbose_name=u'银行卡类型')
cardStatus = models.ForeignKey(BankcardStatus,db_column='fstatus_id',verbose_name=u'状态')
regDate = models.DateField(u'登记日期',db_column='freg_date',auto_now_add=True,blank=True)
class Meta:
db_table='t_bankcard'
verbose_name=u'银行卡'
verbose_name_plural=u'银行卡'
def __unicode__(self):
return '%s-%s'%(self.cardNo,self.name)
def get_absolute_url(self):
return '/bankcard/bankcards/%i'%self.id
"""其他代码略"""
相应Form代码:
class BankcardImportMixin(ModelFormMixin,ImportMixin):
field_mapper = (None,0,1,2,None,3)#映射规则
line_pre_processor = ('trim','unicode','replace_xOO',)#行预处理器
file_pre_processor = ('bom',)#文件预处理器
line_regex=r'(\d{19},[^,]+,[^,]+,(\d{4}-\d{2}-\d{2}|\d{8}))'#检验正则
is_ignore=True#是否忽略错误行
def get_form_class(self):
class form_class(forms.ModelForm):
file = forms.FileField()
class Meta:
model=self.model
exclude = ('cardNo', 'name', 'cardType', 'cardStatus', 'regDate')
return form_class
def form_valid(self,form):
if self.request.method.upper() == "POST":
file = self.request.FILES['file']
try:
instance = self.import_from_file(file)
if instance:
form.instance = instance
else:
raise Exception,u'nothing is imported,please modify file'
except Exception,e:
form.errors['message']=':%s'%e
return super(BankcardImportMixin,self).form_invalid(form)
return super(BankcardImportMixin,self).form_valid(form)
def cardno_converter(self,cardno):
if self.model.objects.filter(cardNo=cardno):
raise Ignore,u'the bankcard no %s already exists'%cardno
def cardtype_converter(self,typename):
model = self.model._meta.get_field_by_name('cardType')[0].rel.to
return model.objects.get(name=typename)
def cardstatus_converter(self,statusname):
model = self.model._meta.get_field_by_name('cardStatus')[0].rel.to
return model.objects.get(name=statusname)
def cardstatus_default(self):
model = self.model._meta.get_field_by_name('cardStatus')[0].rel.to
return model.objects.get(name=u'reg')
View代码:
class BankcardImportView(CreateView,BankcardImportMixin):
"""导入成功显示最后一张卡,全部未成功则转入输入页面"""
model = Bankcard
template_name='upload.html'