python中的pickle是什么意思_python中的pickle问题?

import os

import pickle

from bisect import insort_left,bisect_right

from operator import itemgetter

class Jpickle():

def __init__(self,filename):

logfile=filename+".log"

self.datafile=filename+".dat"

self.indexfile=filename+".index"

self.emptydic={}

self.emptykeys=[]

self.cursorcurrent=0

self.init_status=False

try:

self.data=open(self.datafile,"rb+")

self.index=open(self.indexfile,"rb+")

self.log=open(logfile,"rb+",buffering=0)

self.dic=pickle.loads(self.index.read())

self.cursorfinish=os.path.getsize(self.datafile)

if os.path.getsize(logfile):

while True:

try: i=pickle.load(self.log)

except EOFError: break

sign,argus=i

if sign==4: self.attend(*argus)

elif sign==3: self.dump(*argus)

elif sign==0: self.insert(*argus)

elif sign==1: self.remove(*argus)

elif sign==2: self.dele(*argus)

self.__writeindex()

if self.dic:

var_list=[]

for i in self.dic.values():

if isinstance(i,tuple): var_list.append(i)

else: var_list.extend(i[1:])

var_list.sort(key=itemgetter(0))

cursor_begin=0

for i in var_list:

cursor_first,cursor_second=i

self.__appendemptydic(cursor_first-cursor_begin,cursor_begin)

cursor_begin=cursor_second

self.__appendemptydic(self.cursorfinish-cursor_second,cursor_second)

except FileNotFoundError:

self.dic={}

self.cursorfinish=0

data=open(self.datafile,"wb").close()

index=open(self.indexfile,"wb").close()

log=open(logfile,"wb").close()

self.data=open(self.datafile,"rb+")

self.index=open(self.indexfile,"rb+")

self.log=open(logfile,"rb+",buffering=0)

self.index.write(pickle.dumps(self.dic))

self.index.flush()

self.init_status=True

def __binwrite(self,value):

value_len=len(value)

if self.emptykeys:

empty_len=0

group=value_len//10

plus_pos=bisect_right(self.emptykeys,group)

if plus_pos!=len(self.emptykeys):

group_plus=self.emptykeys[plus_pos]

empty_len,cursor_first=self.emptydic[group_plus].pop(-1)

if not self.emptydic[group_plus]: del self.emptydic[group_plus],self.emptykeys[plus_pos]

else:

if group in self.emptydic:

empty_len,cursor_first=self.emptydic[group][-1]

if empty_len

else:

del self.emptydic[group][-1]

if not self.emptydic[group]: del self.emptydic[group],self.emptykeys[plus_pos-1]

if empty_len:

self.__dseek(cursor_first)

self.__dwrite(value,value_len)

self.__appendemptydic(empty_len-value_len,self.cursorcurrent)

return cursor_first,self.cursorcurrent

self.__dseek(self.cursorfinish)

cursor_first=self.cursorfinish

self.__dwrite(value,value_len)

return cursor_first,self.cursorfinish

def __appendemptydic(self,empty_len,cursor_first):

if empty_len:

group=empty_len//10

if group in self.emptydic: self.emptydic[group].append((empty_len,cursor_first))

else:

self.emptydic[group]=[(empty_len,cursor_first)]

insort_left(self.emptykeys,group)

def __dwrite(self,value,value_len):

self.data.write(value)

self.cursorcurrent+=value_len

if self.cursorcurrent>self.cursorfinish: self.cursorfinish=self.cursorcurrent

def __dseek(self,target):

if self.cursorcurrent!=target:

self.data.seek(target)

self.cursorcurrent=target

def __dread(self,size):

self.cursorcurrent+=size

return self.data.read(size)

def dump(self,var,value):

if var in self.dic: self.dele(var,record=False)

if isinstance(value,tuple):

cursor_first,cursor_second=self.__binwrite(pickle.dumps(value))

self.dic[var]=(cursor_first,cursor_second)

else:

value_write=b"".join(pickle.dumps(i) for i in value)

cursor_first,cursor_second=self.__binwrite(value_write)

self.dic[var]=[len(value),(cursor_first,cursor_second)]

if self.init_status: self.log.write(pickle.dumps((3,(var,value))))

def del(self,var,record=True):

if isinstance(self.dic[var],tuple):

cursor_first,cursor_second=self.dic[var]

self.__appendemptydic(cursor_second-cursor_first,cursor_first)

else:

for i in self.dic[var][1:]:

cursor_first,cursor_second=i

self.__appendemptydic(cursor_second-cursor_first,cursor_first)

del self.dic[var]

if record:

if self.init_status: self.log.write(pickle.dumps((2,(var,))))

def insert(self,var,position,value):

cursor_first,cursor_second,suffix=self.__find(var,position)

ori_first,ori_second=self.dic[var][suffix]

write_first,write_second=self.__binwrite(pickle.dumps(value))

if cursor_first==ori_first:

self.dic[var].insert(suffix,(write_first,write_second))

change_len=1

else:

self.dic[var][suffix]=(ori_first,cursor_first)

self.dic[var].insert(suffix+1,(cursor_first,ori_second))

self.dic[var].insert(suffix+1,(write_first,write_second))

self.dic[var][0]+=1

if self.init_status: self.log.write(pickle.dumps((0,(var,position,value))))

def append(self,var,value):

cursor_first,cursor_second=self.__binwrite(pickle.dumps(value))

if cursor_first==self.dic[var][-1][-1]: self.dic[var][-1]=(self.dic[var][-1][0],cursor_second)

else: self.dic[var].append((cursor_first,cursor_second))

if self.init_status: self.log.write(pickle.dumps((4,(var,value))))

def __find(self,var,position):

value_extent=self.dic[var][0]

if position>value_extent//3*2:

for i in self.generator(var, load_len=value_extent-position, input_status=False): pass

return i

else:

position_search=-1

suffix=0

for i in self.dic[var][1:]:

suffix+=1

cursor_first,cursor_second=i

self.__dseek(cursor_first)

print(self.data.tell())

lock=0

while True:

i=self.__dread(1)

if lock==0 and i==b".": lock+=1

elif lock==1 and i==b"\x80": lock+=1

elif lock==2 and i==b"\x03": lock+=1

elif self.cursorcurrent==cursor_second: return cursor_first,self.cursorcurrent,suffix

else: lock=0

if lock==3:

position_search+=1

if position_search==position: return cursor_first,self.cursorcurrent-2,suffix

else:

lock=0

cursor_first=self.cursorcurrent-2

def remove(self,var,position):

cursor_first,cursor_second,suffix=self.__find(var, position)

ori_first,ori_second=self.dic[var][suffix]

if cursor_first==ori_first and cursor_second==ori_second:

del self.dic[var][suffix]

elif cursor_first==ori_first: self.dic[var][suffix]=(cursor_second,ori_second)

elif cursor_second==ori_second: self.dic[var][suffix]=(ori_first,cursor_first)

else:

self.dic[var][suffix]=(ori_first,cursor_first)

self.dic[var].insert(suffix+1,(cursor_second,ori_second))

if self.init_status: self.log.write(pickle.dumps((1,(var,position))))

def load(self,var):

if isinstance(self.dic[var],tuple):

cursor_first,cursor_second=self.dic[var]

self.__dseek(cursor_first)

value=pickle.loads(self.__dread(cursor_second-cursor_first))

else:

value=[]

for i in self.dic[var][1:]:

cursor_first,cursor_second=i

self.__dseek(cursor_first)

while self.data.tell()!=cursor_second: value.append(pickle.load(self.data))

self.cursorcurrent=cursor_second

return value

def generator(self,var,load_len=-1,input_status=True):

BLOCK=4096

load_number=0

suffix=len(self.dic[var])

for i in reversed(self.dic[var][1:]):

value_search=b""

suffix-=1

cursor_first,cursor_second=i

read_times,lastread_size=divmod(cursor_second-cursor_first,BLOCK)

self.__dseek(cursor_second)

for m in range(read_times):

self.__dseek(self.cursorcurrent-BLOCK)

value_search=self.__dread(BLOCK)+value_search

self.cursorcurrent-=BLOCK

match_finish=len(value_search)

while True:

match_begin=value_search.rfind(b".\x80\x03",0,match_finish)

if match_begin==-1:

value_search=value_search[:match_finish]

break

if input_status: value_yield=pickle.loads(value_search[match_begin+1:match_finish])

else:

value_yield=(self.cursorcurrent+match_begin+1,

self.cursorcurrent+match_finish,

suffix)

match_finish=match_begin+1

yield value_yield

load_number+=1

if load_number==load_len:

self.cursorcurrent+=BLOCK

return

if lastread_size:

self.__dseek(self.cursorcurrent-lastread_size)

value_search=self.__dread(lastread_size)+value_search

self.cursorcurrent-=lastread_size

match_finish=len(value_search)

while True:

match_begin=value_search.rfind(b".\x80\x03",0,match_finish)

if match_begin==-1:

value_search=value_search[:match_finish]

break

if input_status: value_yield=pickle.loads(value_search[match_begin+1:match_finish])

else:

value_yield=(self.cursorcurrent+match_begin+1,

self.cursorcurrent+match_finish,

suffix)

match_finish=match_begin+1

yield value_yield

load_number+=1

if load_number==load_len:

self.cursorcurrent+=lastread_size

return

if input_status: yield pickle.loads(value_search)

else: yield self.cursorcurrent,self.cursorcurrent+match_finish,suffix

self.cursorcurrent+=lastread_size

def dbclear(self):

accept_len=len(self.dic)//3

extend_items=sum(len(x[1:]) for x in self.dic.values() if isinstance(x,list))

if accept_len>extend_items: self.__writeindex()

else:

data_temp=open("dtemp","wb")

new_dic={}

write_first=write_second=0

for i in self.dic.items():

var,value=i

if isinstance(value,tuple):

cursor_first,cursor_second=value

self.__dseek(cursor_first)

data_size=cursor_second-cursor_first

data_temp.write(self.__dread(data_size))

write_second=write_first+data_size

new_dic[var]=(write_first,write_second)

else:

value_extent=self.dic[var][0]

for m in value[1:]:

cursor_first,cursor_second=m

self.__dseek(cursor_first)

data_size=cursor_second-cursor_first

data_temp.write(self.__dread(data_size))

write_second+=data_size

new_dic[var]=[value_extent,(write_first,write_second)]

write_first=write_second

self.data.close()

data_temp.close()

self.dic=new_dic

self.__writeindex()

os.remove(self.datafile)

os.rename("dtemp",self.datafile)

self.data=open(self.datafile,"rb+")

self.cursorcurrent=0

self.cursorfinish=os.path.getsize(self.datafile)

def __writeindex(self):

index_temp=open("itemp","wb")

index_temp.write(pickle.dumps(self.dic))

index_temp.close()

self.index.close()

os.remove(self.indexfile)

os.rename("itemp",self.indexfile)

self.index=open(self.indexfile,"rb+")

self.log.seek(0)

self.log.truncate()