crypto-haystack(BSidesSF ctf 2020)

re和crypto结合的一题，记录一下。

This vendor claims they have figured out a way to preserve the integrity and confidentiality of a message using signing instead of encryption. We only have a binary pycache file and a message off the wire – can you find the content of the message?

大概的意思是本题是关于某种签名算法。
题目本身没有源代码，反编译pyc可执行程序以后可以得到以下代码：

# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.8.7 (tags/v3.8.7:6503f05, Dec 21 2020, 17:59:51) [MSC v.1928 64 bit (AMD64)]
# Embedded file name: /home/david/Projects/BSidesCTF/2020/challenges/haystack/challenge/chaffing.py
# Compiled at: 2020-01-22 09:37:26
# Size of source mod 2**32: 2094 bytes
import hmac, hashlib, random, struct
CHAFF_SIZE = 32
SIG_SIZE = 16
ALL_BYTES = set((c for c in range(256)))
KEY = 'af5f76f605a700ae8c0895c3e6175909'

def byte(v):
    return bytes([v])


def sign_byte(val, key):
    return hmac.new(key,
      val, digestmod=(hashlib.sha256)).digest()[:SIG_SIZE]


def chaff_byte(val, key):
    msgs = {}
    msgs[val[0]] = sign_byte(val, key)
    while len(msgs) < CHAFF_SIZE:
        vals = list(ALL_BYTES - set(msgs.keys()))
        c = random.choice(vals)
        if c == val:
            raise ValueError('Chose duplicate!')
        fake_sig = bytes(random.choices((list(ALL_BYTES)), k=SIG_SIZE))
        msgs[c] = fake_sig

    pieces = []
    for k, v in msgs.items():
        pieces.append('%s%s' % (byte(k), v))

    random.shuffle(pieces)
    return ''.join(pieces)


def chaff_msg(val, key):
    if not isinstance(val, bytes):
        val = val.encode('utf-8')
    msg_out = []
    for b in val:
        msg_out.append(chaff_byte(byte(b), key))

    outval = ''.join(msg_out)
    return struct.pack('>I', len(val)) + outval


def winnow_msg(val, key):
    if not isinstance(val, bytes):
        val = val.encode('utf-8')
    msglen = struct.unpack('>I', val[:4])[0]
    val = val[4:]
    chunk_len = (SIG_SIZE + 1) * CHAFF_SIZE
    expected_len = chunk_len * msglen
    if len(val) != expected_len:
        raise ValueError('Expected length %d, saw %d.' % (expected_len, len(val)))
    pieces = []
    for c in range(msglen):
        chunk = val[chunk_len * c:chunk_len * (c + 1)]
        res = winnow_byte(chunk, key)
        pieces.append(res)

    return ''.join(pieces)


def winnow_byte(val, key):
    while val:
        c = byte(val[0])
        sig = val[1:SIG_SIZE + 1]
        if sign_byte(c, key) == sig:
            return c
        val = val[SIG_SIZE + 1:]

    raise ValueError('No valid sig found!')


def main():
    inp = 'This is a test message!'
    msg = chaff_msg(inp, KEY)
    ret = winnow_msg(msg, KEY)
    if inp != ret:
        print('Wrong ret: %s' % ret)


if __name__ == '__main__':
    main()
# okay decompiling chaffing.pyc

另外题目还提供了一个pcap抓包文件，里面有两个tcp流，将比较大的流内容二进制stream.raw保存下来待用。wireshark保存raw数据的方法参考这里

题目分析

加密部分

1. 消息通过chaff_msg进行加密，对明文的每个字节进行chaff_byte操作
2. 每个字节进行一次 sha256 带密钥key的签名，取签名结果前16字节保存在msgs集合中
3. 然后取另外31个随机字节，每个随机字节生成一个随机假签名也保存在msgs集合中
4. 最后32对 字节+签名 组合进行随机打乱顺序，输出成为密文

解密部分

1. 解析出32对 字节+签名 组合
2. 针对每一对组合，计算当前字节对应的sha256签名，如果计算结果与保存签名不一致的说明不是真实字节值

解题思路

在破译过程中我们不掌握sha256签名时用的key值，所以不能直接计算每个字节的签名值。但是在每个明文字符加密的过程中，特定字符例如字符 a 的签名结果 sig(a) 是不变的，而其他的 假签名 值是完全随机的。
于是我们可以将抓包获取的 字节+签名 组合情况进行统计，真实字符和签名值出现的比例肯定显著高于假字符签名组合。

所以可以用以下方法将每个32对 字节+签名 组合解析出来：

def extract(val):
    if not isinstance(val, bytes):
        val = val.encode('utf-8')
    msglen = struct.unpack('>I', val[:4])[0]
    val = val[4:]
    chunk_len = (SIG_SIZE + 1) * CHAFF_SIZE
    expected_len = chunk_len * msglen
    if len(val) != expected_len:
        raise ValueError('Expected length %d, saw %d.' % (expected_len, len(val)))
    pieces = []
    for c in range(msglen):
        chunk = val[chunk_len * c:chunk_len * (c + 1)]
        res = extract_byte_sig_pairs(chunk)
        pieces.extend(res)
    return pieces
def extract_byte_sig_pairs(val):
    res = []
    while val:
        c = byte(val[0])
        sig = val[1:SIG_SIZE + 1]
        res.append((c, sig))
        val = val[SIG_SIZE + 1:]
    return res

然后取所有组合中出现比例最高的256组，作为真实的 字节+签名 组合对照表：

msg = open('data.bin', 'rb').read()
ret = extract(msg)
c = Counter(ret)
real = c.most_common(256)
print(real)

接下来解密逻辑就很简单了。
构造一个映射关系 d = {s: b for (b, s), c in real} 从32对 字节+签名 中选择真实的一对：

def decode(val, d):
    if not isinstance(val, bytes):
        val = val.encode('utf-8')
    msglen = struct.unpack('>I', val[:4])[0]
    val = val[4:]
    chunk_len = (SIG_SIZE + 1) * CHAFF_SIZE
    expected_len = chunk_len * msglen
    if len(val) != expected_len:
        raise ValueError('Expected length %d, saw %d.' % (expected_len, len(val)))
    pieces = []
    for c in range(msglen):
        chunk = val[chunk_len * c:chunk_len * (c + 1)]
        res = decode_byte(chunk, d)
        pieces.append(res)
    return b''.join(pieces)
def decode_byte(val, d):
    while val:
        c = byte(val[0])
        sig = val[1:SIG_SIZE + 1]
        if sig in d and d[sig] == c:
            return c
        val = val[SIG_SIZE + 1:]
    raise ValueError("WTF")

最终exp：

#20221216
import hmac, hashlib, random, struct
from collections import Counter
CHAFF_SIZE = 32
SIG_SIZE = 16
ALL_BYTES = set((c for c in range(256)))
def extract(val):
    if not isinstance(val, bytes):
        val = val.encode('utf-8')
    msglen = struct.unpack('>I', val[:4])[0]
    val = val[4:]
    chunk_len = (SIG_SIZE + 1) * CHAFF_SIZE
    expected_len = chunk_len * msglen
    if len(val) != expected_len:
        raise ValueError('Expected length %d, saw %d.' % (expected_len, len(val)))
    pieces = []
    for c in range(msglen):
        chunk = val[chunk_len * c:chunk_len * (c + 1)]
        res = extract_byte_sig_pairs(chunk)
        pieces.extend(res)
    return pieces


def extract_byte_sig_pairs(val):
    res = []
    while val:
        c = (val[0])
        sig = val[1:SIG_SIZE + 1]
        res.append((c, sig))
        val = val[SIG_SIZE + 1:]
    return res

msg = open('stream.raw', 'rb').read()
ret = extract(msg)
c = Counter(ret)
real = c.most_common(256)
print(real)


def decode(val, d):
    if not isinstance(val, bytes):
        val = val.encode('utf-8')
    msglen = struct.unpack('>I', val[:4])[0]
    val = val[4:]
    chunk_len = (SIG_SIZE + 1) * CHAFF_SIZE
    expected_len = chunk_len * msglen
    if len(val) != expected_len:
        raise ValueError('Expected length %d, saw %d.' % (expected_len, len(val)))
    pieces = []
    for c in range(msglen):
        chunk = val[chunk_len * c:chunk_len * (c + 1)]
        res = decode_byte(chunk, d)
        pieces.append(res)
    return ''.join(chr(i) for i in pieces)


def decode_byte(val, d):
    while val:
        c = (val[0])
        sig = val[1:SIG_SIZE + 1]
        if sig in d and d[sig] == c:
            return c
        val = val[SIG_SIZE + 1:]
    raise ValueError("WTF")

d = {s: b for (b, s), c in real}
print('\n')
print('\n')
print('\n')
print(d)
print(decode(msg,d))