原始数据:
[-1.13443950e-01 -1.45878432e-01 -1.79049990e-01 -2.12539910e-01 -2.45928442e-01 -2.78799573e-01 -3.10745720e-01 -3.41372281e-01 -3.70301996e-01 -3.97179073e-01 -4.21673013e-01 -4.43482116e-01 -4.62336608e-01 -4.78001364e-01 -4.90278207e-01 -4.99007732e-01 -5.04070674e-01 -5.05388780e-01 -5.02925189e-01 -4.96684324e-01 -4.86711297e-01 -4.73090844e-01 -4.55945795e-01 -4.35435125e-01 -4.11751590e-01 -3.85118995e-01 -3.55789131e-01 -3.24038412e-01 -2.90164261e-01 -2.54481303e-01 -2.17317385e-01 -1.79009507e-01 -1.39899688e-01 -1.00330827e-01 -6.06426160e-02 -2.11675442e-02 1.77729578e-02 5.58721816e-02 9.28415780e-02 1.28413923e-01 1.62346180e-01 1.94422024e-01 2.24454010e-01 2.52285342e-01 2.77791257e-01 3.00879980e-01 3.21493271e-01 3.39606533e-01 3.55228520e-01 3.68400620e-01 3.79195746e-01 3.87716854e-01 3.94095097e-01 3.98487660e-01 4.01075290e-01 4.02059569e-01 4.01659961e-01 4.00110663e-01 3.97657329e-01 3.94553671e-01 3.91058019e-01 3.87429850e-01 3.83926361e-01 3.80799096e-01 3.78290699e-01 3.76631809e-01 3.76038146e-01 3.76707824e-01 3.78818907e-01 3.82527254e-01 3.87964672e-01 3.95237379e-01 4.04219226e-01 4.13343882e-01 4.24570706e-01 4.37987680e-01 4.53655629e-01 4.71606864e-01 4.91844150e-01 5.14340045e-01 5.39036583e-01 5.65845351e-01 5.94647928e-01 6.25296720e-01 6.57616151e-01 6.91404228e-01 7.26434455e-01 7.62458071e-01 7.99206597e-01 8.36394668e-01 8.73723103e-01 9.10882193e-01 9.47555167e-01 9.83421789e-01 1.01816205e+00 1.05145989e+00 1.08300697e+00 1.11250637e+00 1.13967614e+00 1.16425286e+00 1.18599490e+00 1.20468549e+00 1.22013554e+00 1.23218620e+00 1.24071101e+00 1.24561775e+00 1.24684986e+00 1.24438750e+00 1.23824816e+00 1.22848680e+00 1.21519558e+00 1.19850317e+00 1.17857355e+00 1.15560445e+00 1.12982534e+00 1.10149497e+00 1.07089869e+00 1.03834527e+00 1.00416350e+00 9.68698523e-01 9.32307897e-01 8.95357548e-01 8.58217534e-01 8.21257756e-01 7.84843644e-01 7.49331856e-01 7.15066061e-01 6.82372851e-01 6.51557835e-01 6.22901955e-01 5.96658089e-01 5.73047966e-01 5.52259442e-01 5.34444181e-01 5.19715751e-01 5.08148186e-01 4.99775015e-01 4.94588789e-01 4.92541107e-01 4.93543144e-01 4.97466683e-01 5.04145641e-01 5.13378069e-01 5.24928616e-01 5.38531414e-01 5.53893371e-01 5.70697813e-01 5.88608458e-01 6.07273649e-01 6.26330814e-01 6.45411098e-01 6.64144099e-01 6.82162665e-01 6.99107684e-01 7.14632813e-01 7.28409081e-01 7.40129312e-01 7.49512315e-01 7.56306779e-01 7.60294830e-01 7.61295197e-01 7.59165950e-01 7.53806768e-01 7.45160709e-01 7.33215447e-01 7.18003975e-01 6.99604733e-01 6.78141180e-01 6.53780797e-01 6.26733525e-01 5.97249669e-01 5.65617266e-01 5.32158960e-01 4.97228422e-01 4.61206333e-01 4.24495997e-01 3.87518627e-01 3.50708354e-01 3.14507024e-01 2.79358841e-01 2.45704919e-01 2.13977816e-01 1.84596107e-01 1.57959068e-01 1.34441531e-01 1.14388987e-01 9.81129815e-02 8.58868749e-02 7.79420140e-02 7.44643694e-02 7.55916818e-02 8.14111553e-02 9.19577331e-02 1.07212978e-01 1.27104580e-01 1.51506499e-01 1.80239749e-01 2.13073818e-01 2.49728710e-01 2.89877600e-01 3.33150056e-01 3.79135814e-01 4.27389050e-01 4.77433117e-01 5.28765669e-01 5.80864151e-01 6.33191546e-01 6.85202361e-01 7.36348736e-01 7.86086643e-01 8.33882074e-01 8.79217158e-01 9.21596127e-01 9.60551067e-01 9.95647374e-01 1.02648885e+00 1.05272238e+00 1.07404215e+00 1.09019329e+00 1.10097495e+00 1.10624281e+00 1.10591087e+00 1.09995258e+00 1.08840127e+00 1.07134993e+00 1.04895019e+00 1.02141068e+00 9.88994652e-01 9.52017015e-01 9.10840667e-01 8.65872294e-01 8.17557617e-01 7.66376160e-01 7.12835592e-01 6.57465720e-01 6.00812198e-01 5.43430021e-01 4.85876887e-01 4.28706516e-01 3.72461982e-01 3.17669166e-01 2.64830392e-01 2.14418332e-01 1.66870247e-01 1.22582650e-01 8.19064436e-02 4.51426021e-02 1.25384474e-02 -1.57154267e-02 -3.94875454e-02 -5.87072248e-02 -7.33655017e-02 -8.35153048e-02 -8.92708620e-02 -9.08063466e-02 -8.83537756e-02 -8.22001808e-02 -7.26840830e-02 -6.01913103e-02 -4.51502053e-02 -2.80262787e-02 -9.31636987e-03 1.04576164e-02 3.07553231e-02 5.10248252e-02 7.07097027e-02 8.92562169e-02 1.06120503e-01 1.20775690e-01 1.32718866e-01 1.41477789e-01 1.46617282e-01 1.47745207e-01 1.44517962e-01 1.36645422e-01 1.23895258e-01 1.06096585e-01 8.31428812e-02 5.49941477e-02 2.16782642e-02 -1.67084681e-02 -6.00016169e-02 -1.07969742e-01 -1.60316346e-01 -2.16682629e-01 -2.76651026e-01 -3.39749490e-01 -4.05456464e-01 -4.73206505e-01 -5.42396474e-01 -6.12392246e-01 -6.82535842e-01 -7.52152911e-01 -8.20560477e-01 -8.87074847e-01 -9.51019604e-01 -1.01173357e+00 -1.06857867e+00 -1.12094756e+00 -1.16827098e+00 -1.21002471e+00 -1.24573604e+00 -1.27498965e+00 -1.29743297e+00 -1.31278067e+00 -1.32081856e+00 -1.32140661e+00 -1.31448107e+00 -1.30005580e+00 -1.27822268e+00 -1.24915106e+00 -1.21308638e+00 -1.17034782e+00 -1.12132514e+00 -1.06647463e+00 -1.00631430e+00 -9.41418331e-01 -8.72410787e-01 -7.99958803e-01 -7.24765181e-01 -6.47560564e-01 -5.69095254e-01 -4.90130770e-01 -4.11431247e-01 -3.33754776e-01 -2.57844790e-01 -1.84421600e-01 -1.14174178e-01 -4.77522851e-02 1.42409504e-02 7.12559285e-02 1.22802852e-01 1.68457307e-01 2.07865082e-01 2.40746158e-01 2.66897828e-01 2.86196906e-01 2.98600977e-01 3.04148702e-01 3.02959140e-01 2.95230116e-01 2.81235628e-01 2.61322348e-01 2.35905232e-01 2.05462300e-01 1.70528651e-01 1.31689772e-01 8.95742207e-02 4.48457790e-02 -1.80484198e-03 -4.96686404e-02 -9.80272036e-02 -1.46161723e-01 -1.93362035e-01 -2.38935577e-01 -2.82216150e-01 -3.22572388e-01 -3.59415811e-01 -3.92208389e-01 -4.20469493e-01 -4.43782176e-01 -4.61798684e-01 -4.74245125e-01 -4.80925260e-01 -4.81723335e-01 -4.76605936e-01 -4.65622832e-01 -4.48906792e-01 -4.26672374e-01 -3.99213699e-01 -3.66901217e-01 -3.30177528e-01 -2.89552272e-01 -2.45596168e-01 -1.98934261e-01 -1.50238453e-01 -1.00219414e-01 -4.96179540e-02 8.04027227e-04 5.02729168e-02 9.80129741e-02 1.43256011e-01 1.85251054e-01 2.23273862e-01 2.56636195e-01 2.84694718e-01 3.06859417e-01 3.22601446e-01 3.31460288e-01 3.33050148e-01 3.27065497e-01 3.13285691e-01 2.91578609e-01 2.61903261e-01 2.24311321e-01 1.78947569e-01 1.26049232e-01 6.59442095e-02 -9.51774445e-04 -7.41390931e-02 -1.53039152e-01 -2.37000291e-01 -3.25304590e-01 -4.17175501e-01 -5.11786210e-01 -6.08268658e-01 -7.05723084e-01 -8.03228012e-01 -8.99850536e-01 -9.94656811e-01 -1.08672260e+00 -1.17514378e+00 -1.25904665e+00 -1.33759792e+00 -1.41001436e+00 -1.47557177e+00 -1.53361345e+00 -1.58355780e+00 -1.62490515e+00 -1.65724364e+00 -1.68025406e+00 -1.69371373e+00 -1.69749912e+00 -1.69158748e+00 -1.67605717e+00 -1.65108691e+00 -1.61695378e+00 -1.57403012e+00 -1.52277927e+00 -1.46375032e+00 -1.39757168e+00 -1.32494392e+00 -1.24663160e+00 -1.16345437e+00 -1.07627746e+00 -9.86001572e-01 -8.93552372e-01 -7.99869649e-01 -7.05896329e-01 -6.12567429e-01 -5.20799095e-01 -4.31477871e-01 -3.45450298e-01 -2.63513005e-01 -1.86403375e-01 -1.14790927e-01 -4.92695084e-02 9.64960961e-03 6.15436418e-02 1.06083134e-01 1.43035679e-01 1.72268465e-01 1.93749607e-01 2.07548257e-01 2.13833481e-01 2.12871916e-01 2.05024227e-01 1.90740411e-01 1.70553992e-01 1.45075189e-01 1.14983118e-01 8.10171369e-02 4.39674180e-02 4.66488072e-03 -3.60294131e-02 -7.72352662e-02 -1.18064624e-01 -1.57632986e-01 -1.95070823e-01 -2.29534836e-01 -2.60218959e-01 -2.86364925e-01 -3.07272309e-01 -3.22307887e-01 -3.30914222e-01 -3.32617355e-01 -3.27033502e-01 -3.13874687e-01 -2.92953217e-01 -2.64184959e-01 -2.27591357e-01 -1.83300175e-01 -1.31544933e-01 -7.26630546e-02 -7.09273125e-03 6.46314656e-02 1.41884170e-01 2.23955950e-01 3.10061101e-01 3.99346417e-01 4.90900832e-01 5.83765837e-01 6.76946529e-01 7.69423182e-01 8.60163188e-01 9.48133244e-01 1.03231162e+00 1.11170039e+00 1.18533742e+00 1.25230809e+00 1.31175642e+00 1.36289569e+00 1.40501823e+00 1.43750433e+00 1.45983026e+00 1.47157511e+00 1.47242651e+00 1.46218509e+00 1.44076767e+00 1.40820908e+00 1.36466262e+00 1.31039912e+00 1.24580467e+00 1.17137695e+00 1.08772027e+00 9.95539322e-01 8.95631772e-01 7.88879717e-01 6.76240135e-01 5.58734433e-01 4.37437213e-01 3.13464375e-01 1.87960720e-01 6.20871691e-02 -6.29922312e-02 -1.86123369e-01 -3.06175046e-01 -4.22051710e-01 -5.32705796e-01 -6.37149537e-01 -7.34466081e-01 -8.23819804e-01 -9.04465682e-01 -9.75757617e-01 -1.03715560e+00 -1.08823166e+00 -1.12867444e+00 -1.15829251e+00 -1.17701615e+00 -1.18489778e+00 -1.18211091e+00 -1.16894766e+00 -1.14581490e+00 -1.11322898e+00 -1.07180919e+00 -1.02227000e+00 -9.65412131e-01 -9.02112624e-01 -8.33313985e-01 -7.60012539e-01 -6.83246146e-01 -6.04081409e-01 -5.23600539e-01 -4.42888027e-01 -3.63017283e-01 -2.85037412e-01 -2.09960272e-01 -1.38747985e-01 -7.23010423e-02 -1.14471539e-02 4.30690206e-02 9.05951381e-02 1.30579316e-01 1.62577054e-01 1.86256853e-01 2.01404444e-01 2.07925595e-01 2.05847432e-01 1.95318281e-01 1.76606003e-01 1.50094858e-01 1.16280917e-01 7.57660840e-02 2.92507909e-02 -2.24745546e-02 -7.85392586e-02 -1.36150080e-01 -1.95662574e-01 -2.56493366e-01 -3.17567385e-01 -3.77790504e-01 -4.36063310e-01 -4.91295016e-01 -5.42417336e-01 -5.88398164e-01 -6.28254886e-01 -6.61067162e-01 -6.85989028e-01 -7.02260162e-01 -7.09216176e-01 -7.06297810e-01 -6.93058907e-01 -6.69173070e-01 -6.34438919e-01 -5.88783869e-01 -5.32266392e-01 -4.65076705e-01 -3.87535903e-01 -3.00093506e-01 -2.03323464e-01 -9.79186525e-02 1.53160754e-02 1.35472218e-01 2.61547224e-01 3.92455266e-01 5.27039063e-01 6.64082602e-01 8.02324638e-01 9.40472787e-01 1.07721807e+00 1.21124972e+00 1.34127007e+00 1.46600944e+00 1.58424060e+00 1.69479303e+00 1.79656636e+00 1.88854320e+00 1.96980101e+00 2.03952290e+00 2.09700723e+00 2.14167595e+00 2.17308151e+00 2.19091223e+00 2.19499622e+00 2.18530356e+00 2.16194690e+00 2.12518046e+00 2.07539724e+00 2.01312478e+00 1.93901921e+00 1.85385788e+00 1.75853054e+00 1.65402922e+00 1.54143685e+00 1.42191491e+00 1.29669003e+00 1.16703997e+00 1.03427883e+00 8.99741997e-01 7.64770686e-01 6.30696537e-01 4.98826252e-01 3.70426558e-01 2.46709640e-01 1.28819225e-01 1.78174813e-02 -8.53271007e-02 -1.79750717e-01 -2.64704000e-01 -3.39560395e-01 -4.03823070e-01 -4.57130259e-01 -4.99258991e-01 -5.30127137e-01 -5.49793770e-01 -5.58457812e-01 -5.56454994e-01 -5.44253164e-01 -5.22445981e-01 -4.91745092e-01 -4.52970864e-01 -4.07041793e-01 -3.54962709e-01 -2.97811918e-01 -2.36727437e-01 -1.72892485e-01 -1.07520406e-01 -4.18392007e-02 2.29241429e-02 8.55593312e-02 1.44888269e-01 1.99780201e-01 2.49166299e-01 2.92053524e-01 3.27537591e-01 3.54814876e-01 3.73193115e-01 3.82100770e-01 3.81094936e-01 3.69867692e-01 3.48250819e-01 3.16218811e-01 2.73890152e-01 2.21526820e-01 1.59532035e-01 8.84462613e-02 8.94150858e-03 -7.81860030e-02 -1.72024735e-01 -2.71558179e-01 -3.75676531e-01 -4.83189580e-01 -5.92840634e-01 -7.03321330e-01 -8.13287170e-01 -9.21373566e-01 -1.02621223e+00 -1.12644771e+00 -1.22075387e+00 -1.30785009e+00 -1.38651705e+00 -1.45561188e+00 -1.51408244e+00 -1.56098066e+00 -1.59547467e+00 -1.61685966e+00 -1.62456726e+00 -1.61817343e+00 -1.59740456e+00 -1.56214199e+00 -1.51242460e+00 -1.44844962e+00 -1.37057160e+00 -1.27929948e+00 -1.17529191e+00 -1.05935073e+00 -9.32412795e-01 -7.95540176e-01 -6.49908873e-01 -4.96796143e-01] |
压缩后数据:
IO+w!r7w~^Zf8?nfR2cbs==_3kEEM;ccs0SYIBBPBrbK5XHR>SQwSn>id$y9)_5UAjdnY0=l_mDbDmBBGFb |
import numpy as np
from matplotlib import pyplot as plt
import base64
def init_N(n, k, X, T):
"""
生成初始化的参数矩阵
:param n: 控制点数量,int
:param k: 基函数的阶数,int, 默认为2,建议2,3,4,最小为2,越大越平滑,越大速度越慢
:param X: 自变量矩阵,取值范围[0,1] (shape=[sample], dtype=float)
:param T: 控制点系数矩阵,取值范围[0,1] (shape=[n + k + 1], dtype=float)
:return N: (shape=[sample, n], dtype=float)
"""
m = n + k + 1
X = X[None] # shape=[1, sample]
T = T[:, None] # shape=[n+k+1, 1]
m = n + k + 1
N = 1.0 * (T[:-1] <= X) * (X <= T[1:]) # shape=[n+k, sample], dtype=float
for ki in range(1, k + 1): # 动态规划 k ∈ [1, k]
n = m - ki - 1 # 每次更新的控制点数量
c1 = T[ki:n + ki] - T[:n]
c2 = T[ki + 1:n + ki + 1] - T[1:n + 1,]
c1[c1 == 0] = 1 # 替换 c1 等于 0 的情况,如果等于0 则替换成1,否则不变
c2[c2 == 0] = 1 # 替换 c2 等于 0 的情况,如果等于0 则替换成1,否则不变
w1 = (X - T[:n]) / c1
w2 = (T[ki + 1:n + ki + 1] - X) / c2
N = N[:-1] * w1 + N[1:] * w2
N = N[:n].T # 截取前n个,然后交换行列
return N # sample, n
def init_T_X(k, m, sample):
"""
生成初始化的参数向量
:param k: 基函数的阶数,int, 默认为2,建议2,3,4,最小为2,越大越平滑,越大速度越慢
:param m: 系数矩阵的长度,int,m=n+k+1
:param sample: 采样点数量, int
:return T: 系数矩阵,取值范围[0,1] (shape[n + k + 1], dtype=float)
:return X: 自变量矩阵,取值范围[0,1] (shape=[sample], dtype=float)
"""
# clame 模式
T = np.concatenate([
np.zeros(k),
np.linspace(0, 1, m - 2 * k),
np.ones(k)]
)
gama = 1e-9
first = T[k]
last = T[-k - 1] * (1 - gama) + gama * T[-k - 2]
X = np.linspace(first, last, sample)
return T, X
def unzip(P, k, sample):
"""
解压点,生成曲线
:param P: 控制点矩阵,(shape=[n], dtype=float)
:param k: 基函数的阶数,int, 默认为2,建议2,3,4,最小为2,越大越平滑,越大速度越慢
:param sample: 采样点数量, int
:return data: 解压后的曲线,(shape=[sample], dtype=float)
"""
n = len(P)
T, X = init_T_X(k=k, m=k + n + 1, sample=sample)
N = init_N(n=n, k=k, T=T, X=X)
numerator = np.sum(P[None] * N, axis=-1)
denominator = np.sum(N, axis=-1)
data = numerator / denominator
return data
def zip(data, n, k):
"""
压缩曲线,生成控制点矩阵
:param data: 曲线上的采样点,(shape=[sample], dtype=float)
:param n: 控制点的数量,int
:param k: 曲线的阶数,int
:return P: 控制点矩阵, (shape=[n], dtype=float)
"""
sample = len(data)
T, X = init_T_X(k=k, m=k + n + 1, sample=sample)
a = init_N(n=n, k=k, T=T, X=X)
b = data * np.sum(a, axis=-1)
P = np.linalg.lstsq(a, b, rcond=None)[0]
loss = np.sqrt(np.sum((a@P[:,None] - b[:, None])**2)) # 求二范数
print('loss : ', loss)
return P,loss
def plot_n_loss(data, k=3):
ns,losss = [],[]
for i in range(10, 128):
P, loss = zip(data, i, k)
ns.append(i)
losss.append(np.log10(loss))
plt.plot(ns, losss)
plt.show()
def zip_auto(data,
min_n = 4,
max_n = 128,
loss_threshold= 0.1,# 可以接收的最小误差范围
):
"""
自适应压缩曲线,生成控制点矩阵,自动查找合适的n
:param data: 曲线上的采样点,(shape=[sample], dtype=float)
:param min_n: 最小的控制点数量,int
:param max_n: 最大的控制点数量,int
:param min_loss_threshold: 最小的二范数误差,float
:param max_loss_threshold: 最大的二范数误差,float
:return P: 控制点矩阵, (shape=[n], dtype=float)
:return loss: 二范数误差, (shape=[n], dtype=float)
"""
k = 3
l,r = min_n - 1, max_n + 1
n = l
resP = None
resLoss = None
while l < r*0.9: # r越大允许的误差越大,r越小允许的n的误差范围越小
n = (l+r)//2
P, loss = zip(data, n, k)
if loss > loss_threshold:
l = n + 1
else:
resP = P
resLoss = loss
r = n - 1
if not l < r*0.85:
break
if resP is None:
resP = P
resLoss = loss
return resP, resLoss
def unzip_auto(P, sample):
"""
自动解压点,生成曲线
:param P: 控制点矩阵,(shape=[n], dtype=float)
:param sample: 采样点数量, int
:return data: 解压后的曲线,(shape=[sample], dtype=float)
"""
k = 3
n = len(P)
T, X = init_T_X(k=k, m=k + n + 1, sample=sample)
N = init_N(n=n, k=k, T=T, X=X)
numerator = np.sum(P[None] * N, axis=-1)
denominator = np.sum(N, axis=-1)
data = numerator / denominator
return data
class LineBytes:
def __init__(self, half=False):
""" 初始化构造函数
:param half: 是否使用半精度浮点数,bool
"""
if half:
self.dtype = np.float16
self.head_bytes = 4
else:
self.dtype = np.float32
self.head_bytes = 8
def unzip(self, bytes, sample):
head = np.frombuffer(bytes[:self.head_bytes], dtype=self.dtype)
map = np.frombuffer(bytes[self.head_bytes:], dtype=np.uint8).astype(np.float32)
points = head[0] + map * head[1] # 后面的数据做映射解码
data = unzip_auto(points, sample)
return data, points
def zip(self, data):
""" 压缩函数
:param data: 需要压缩的数据列表,(shape=[-1], dtype=float)
:return b85: 压缩后的base85编码的字符串,str
"""
points, loss = zip_auto(data, loss_threshold=0.1)
head = np.array([points.min(), (points.max() - points.min()) / 255.])
map = (points - head[0]) / head[1]
bytes = head.astype(self.dtype).tobytes() + map.astype(np.uint8).tobytes()
return bytes
def unzipb85(self, b85:str, sample=100):
""" 解压缩函数
:param b85: base85编码的字符串,str
:return data: 解压缩后的数据 (shape=[-1], dtype=float)
"""
bytes = base64.b85decode(b85)
return self.unzip(bytes, sample)
def zipb85(self, data):
""" 压缩函数
:param data: 需要压缩的数据列表,(shape=[-1], dtype=float)
:return b85: 压缩后的base85编码的字符串,str
"""
b85 = base64.b85encode(self.zip(data))
return b85
if __name__ == "__main__":
sample = 720 # 采样数量
X = np.linspace(0, 1, sample)
Y = np.sin(3 * np.pi * X + 6) * (0.2 + np.abs(X - 0.8)) + \
np.sin(25 * np.pi * X + 9) * (0.2 + np.abs(X - 0.1)) + \
np.sin(11 * np.pi * X + 3) * (0.2 + np.abs(X - 0.1))
lb = LineBytes()
b85 = lb.zipb85(Y)
unzip_Y, points = lb.unzipb85(b85, sample)
print(Y)
print(len(b85), b85)
chars = b85.decode()
print(len(chars), chars)
print("mean/μ :", Y.mean())
print("var/σ :", Y.var())
print("err/ι :", np.abs(unzip_Y - Y).mean())
plt.plot(X, Y, 'g-', marker='+', label='origin line')
plt.plot(np.linspace(0,1, len(points)), points, 'b-.', marker='o', label='zip points')
plt.plot(X, unzip_Y, 'r-', marker='+', label='unzip line')
plt.legend()
plt.show()