xg03算法核心

前言

随着抖音的不断升级，xg算法作为比较核心的一个算法，也经历过一次又一次的变化，2020年7月份的时候使用03的算法还可以使用，经过几个月的时间不晓得的还能不能，毕竟也没有继续深入研究抖音的算法了，这个行业水太深了，凭借着自己那一点逆向基础可能还不足以跟上抖音大佬们更新迭代的步骤，因此把之前弄的一套xg03的源码分享给大家。仅仅作为学习可以的。

核心

@RestController
public class XGorgonSell {
    protected static final String URL_REGEX = "^((https|http|ftp|rtsp|mms)?://)(([0-9]{1,3}\\.){3}[0-9]{1,3}|([0-9a-z_!~*'()-]+\\.)*([0-9a-z][0-9a-z-]{0,61})?[0-9a-z]\\.[a-z]{2,6})(:[0-9]{1,5})?((/?)|(/[0-9a-z_!~*'().;?:@&=+$,%#-]+)+/?)$";

    public static String byteTable1 =
        "D6 28 3B 71 70 76 BE 1B A4 FE 19 57 5E 6C BC 21 B2 14 37 " +
        "7D 8C A2 FA 67 55 6A 95 E3 FA 67 78 ED 8E 55 33 " +
        "89 A8 CE 36 B3 5C D6 B2 6F 96 C4 34 B9 6A EC 34 " +
        "95 C4 FA 72 FF B8 42 8D FB EC 70 F0 85 46 D8 B2 " +
        "A1 E0 CE AE 4B 7D AE A4 87 CE E3 AC 51 55 C4 36 " +
        "AD FC C4 EA 97 70 6A 85 37 6A C8 68 FA FE B0 33 " +
        "B9 67 7E CE E3 CC 86 D6 9F 76 74 89 E9 DA 9C 78 " +
        "C5 95 AA B0 34 B3 F2 7D B2 A2 ED E0 B5 B6 88 95 " +
        "D1 51 D6 9E 7D D1 C8 F9 B7 70 CC 9C B6 92 C5 FA " +
        "DD 9F 28 DA C7 E0 CA 95 B2 DA 34 97 CE 74 FA 37 " +
        "E9 7D C4 A2 37 FB FA F1 CF AA 89 7D 55 AE 87 BC " +
        "F5 E9 6A C4 68 C7 FA 76 85 14 D0 D0 E5 CE FF 19 " +
        "D6 E5 D6 CC F1 F4 6C E9 E7 89 B2 B7 AE 28 89 BE " +
        "5E DC 87 6C F7 51 F2 67 78 AE B3 4B A2 B3 21 3B " +
        "55 F8 B3 76 B2 CF B3 B3 FF B3 5E 71 7D FA FC FF " +
        "A8 7D FE D8 9C 1B C4 6A F9 88 B5 E5";

    public static String[] byteTable2 = byteTable1.split(" ");
    private static final String NULL_MD5_STRING = "00000000000000000000000000000000";

    public static String cookies = "sid_tt=74c52723180b10978d083d14fced36f2; uid_tt=593003c106c6de4a1198fb10ff9053f4; ttreq=1$85338baf79830fee82d11295117fc897e8d76892; sid_guard=74c52723180b10978d083d14fced36f2%7C1548476024%7C5184000%7CWed%2C+27-Mar-2019+04%3A13%3A44+GMT; odin_tt=7f57a517a48a1d2ad03e716ced77e48c8b03b7aa786c21e21a8a4ab7b4aae1521cf947f5fa106b969fd94ffc6b6003a372470eef06deeb01a8e550e0fc5689aa; install_id=50651589426; sessionid=74c52723180b10978d083d14fced36f2";


    /**
     * 使用案例demo 获取抖音视频详情
     * @param shareUrl
     * @return
     * @throws IOException
     */
    @GetMapping("/douyinDetails222")
    @ResponseBody
    public JSONObject douyinVideoParse(@RequestParam String shareUrl) throws IOException {
        int ts = (int) (System.currentTimeMillis() / 1000);
        String _rticket = System.currentTimeMillis() + "";

        String realUrl = shortUrlConvertLongUrl(shareUrl);

        String aweme_id = matchUrlKeyAfterValue(realUrl,"video");
        String  url = "https://aweme-lq.snssdk.com/aweme/v1/aweme/detail/?aweme_id="+aweme_id+"&origin_type=link&manifest_version_code=700&_rticket="+_rticket+"&app_type=normal&iid=93666507608&channel=goapk_aweme&device_type=MI%206&language=zh&uuid=867391038812504&resolution=1080*1920&openudid=26b3dc2e1d807ca8&update_version_code=7002&os_api=26&dpi=480&ac=wifi&device_id=50624531127&os_version=8.0.0&version_code=700&app_name=aweme&version_name=7.0.0&js_sdk_version=1.18.2.5&device_brand=Xiaomi&ssmix=a&device_platform=android&aid=1128&ts="+ts;
        //cookies
        String cookies = "替换cookie";
        JSONObject jsonObject = new JSONObject();
        String params = url.substring(url.indexOf("?") + 1, url.length());
        String STUB = "";
        String s = getXGon(params, STUB, cookies);
        String Gorgon = xGorgon(ts, StrToByte(s));
        Map<String, Object> headers = new HashMap<>();
        headers.put("X-Gorgon", Gorgon);
        headers.put("X-Khronos", ts);
        headers.put("sdk-version", "1");
        headers.put("Accept-Encoding", "gzip");
        headers.put("X-SS-REQ-TICKET", _rticket);
        headers.put("User-Agent", "com.ss.android.ugc.aweme/700 (Linux; U; Android 8.0.0; zh_CN; MI 6; Build/OPR1.170623.027; Cronet/58.0.2991.0)");
        headers.put("x-Tt-Token", "替换");
        headers.put("Host", "aweme-lq.snssdk.com");
        headers.put("Cookie", cookies);
        headers.put("Connection", "Keep-Alive");
        headers.put("Content-Type", "application/x-www-form-urlencoded;charset=utf-8");
        String json = doGetGzip(url, headers,"UTF-8");

        return JSONObject.parseObject(json);
    }
    

    /**
     * get请求 gzip方式 解压gzip
     * 请求的时候 请求头是这样的就属于gzip headers.put("Accept-Encoding", "gzip");
     * org.apache.commons.io.IOUtils在commons-io中
     * @param url
     * @param headers
     * @param charset
     * @return
     * @throws IOException
     */
    public static String doGetGzip(String url, Map<String, Object> headers, String charset) throws IOException {
        String result = null;
        InputStream is = null;
        try{
            URL realUrl = new URL(url);
            // 打开和URL之间的连接
            HttpURLConnection conn = (HttpURLConnection) realUrl.openConnection();
            // 发送POST请求必须设置如下两行
            conn.setRequestMethod("GET");// 提交模式
            conn.setDoInput(true);
            conn.setDoOutput(false);
            conn.setUseCaches(false);
            if (headers!=null){
                for (Map.Entry<String, Object> entry : headers.entrySet()) {
                    conn.setRequestProperty(entry.getKey(), entry.getValue().toString());
                }
            }
            conn.setRequestProperty("Connection", "close");
            conn.setConnectTimeout(3000);  //设置连接主机超时（单位：毫秒）
            conn.setReadTimeout(3000);     //设置从主机读取数据超时（单位：毫秒）
            String content_encoding = conn.getHeaderField("Content-Encoding");
            is = conn.getInputStream();
            if(content_encoding!=null&&content_encoding.equals("gzip")) {
				/*InputStream stream = new GZIPInputStream(conn.getInputStream());
				BufferedReader reader = new BufferedReader(new InputStreamReader(stream,"utf-8"));
				StringBuffer sb = new StringBuffer();
				String line = "";
				while ((line = reader.readLine()) != null){
					sb.append(line);
				}
				System.out.println(sb.toString());*/
                InputStream stream = new GZIPInputStream(is);
                result = IOUtils.toString(stream, charset);
            }else {
                result =  IOUtils.toString(is, charset);
            }
        }catch (Exception e){
            e.printStackTrace();
        }finally {
            if (is!=null){
                is.close();
            }
        }
        return result;
    }
    /**
     * 获取url中/后面的值
     * 比如我要获取这个链接里面的video后面的值https://www.iesdouyin.com/share/video/65656565655656/?region=CN&mid=4545454545
     * 我把这个链接传递进去，key就传递video 就能获取到后面的值了45454545454
     * @param url url长链接
     * @param key 你要获取的那个key的值
     * @return
     */
    public static String matchUrlKeyAfterValue(String url, String key) {
        String[] param = url.split("/");
        for (int i = 0; i < param.length; i++) {
            String str = param[i];
            String result = param[i + 1];
            if (key.equals(str)) {
                if (str.contains("?")) {
                    return result;
                }
                if (result.contains("?")) {
                    return result.substring(0, result.indexOf("?"));
                }
                return result;
            }
        }
        return null;
    }

    /**
     * @param str url字符串
     * @return boolean
     * @author HONGLINCHEN
     * @description 判断一个字符串是否为url
     * @date 2019-08-20-0020 17:47
     */
    public static boolean isURL(String str) {
        //转换为小写
        str = str.toLowerCase();
        String regex = "^((https|http|ftp|rtsp|mms)?://)"//https、http、ftp、rtsp、mms
                // + "?(([0-9a-z_!~*'().&=+$%-]+: )?[0-9a-z_!~*'().&=+$%-]+@)?" //ftp的user@  
                + "(([0-9]{1,3}\\.){3}[0-9]{1,3}" // IP形式的URL- 例如：199.194.52.184  
                + "|" // 允许IP和DOMAIN（域名）
                + "([0-9a-z_!~*'()-]+\\.)*" // 域名- www.  
                + "([0-9a-z][0-9a-z-]{0,61})?[0-9a-z]\\." // 二级域名  
                + "[a-z]{2,6})" // first level domain- .com or .museum  
                + "(:[0-9]{1,5})?" // 端口号最大为65535,5位数
                + "((/?)|" // a slash isn't required if there is no file name  
                + "(/[0-9a-z_!~*'().;?:@&=+$,%#-]+)+/?)$";
        return str.matches(URL_REGEX);
    }

    /**
     * 通过短链接 获取长链接
     * 短链接转换成长链接
     * @param shortUrl 短链接
     */
    public static String shortUrlConvertLongUrl(String shortUrl){
        if (isURL(shortUrl)) {
            Document document = null;
            try {
                //ignoreHttpErrors  忽略请求错误404 例如当把http://v.douyin.com/jek538/转成长链接的时候会报错 因为这个页面是404  加上这个不会报错
                document = Jsoup.connect(shortUrl).timeout(60000).method(Connection.Method.GET).ignoreHttpErrors(true).followRedirects(true).get();
            } catch (IOException e) {
                e.printStackTrace();
            }
            return  document.location();
        } else {
            return "非法网址";
        }
    }
    public static String ByteToStr(byte[] bArr) {

        int i = 0;

        char[] toCharArray = "0123456789abcdef".toCharArray();
        char[] cArr = new char[(bArr.length * 2)];
        while (i < bArr.length) {
            int i2 = bArr[i] & 255;
            int i3 = i * 2;
            cArr[i3] = toCharArray[i2 >>> 4];
            cArr[i3 + 1] = toCharArray[i2 & 15];
            i++;
        }
        return new String(cArr);

    }
    public static byte[] StrToByte(String str) {
        String str2 = str;
        Object[] objArr = new Object[1];
        int i = 0;
        objArr[0] = str2;

        int length = str.length();
        byte[] bArr = new byte[(length / 2)];
        while (i < length) {
            bArr[i / 2] = (byte) ((Character.digit(str2.charAt(i), 16) << 4) + Character.digit(str2.charAt(i + 1), 16));
            i += 2;
        }
        return bArr;
    }

    public static String encryption(String str) {
        String re_md5 = null;
        try {
            MessageDigest md = MessageDigest.getInstance("MD5");
            md.update(str.getBytes());
            byte b[] = md.digest();

            int i;

            StringBuffer buf = new StringBuffer("");
            for (int offset = 0; offset < b.length; offset++) {
                i = b[offset];
                if (i < 0)
                    i += 256;
                if (i < 16)
                    buf.append("0");
                buf.append(Integer.toHexString(i));
            }

            re_md5 = buf.toString();

        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        }
        return re_md5.toUpperCase();
    }

    public static String getXGon(String url, String stub, String ck) {
        StringBuilder sb = new StringBuilder();
        if (TextUtils.isEmpty(url)) {
            sb.append(NULL_MD5_STRING);
        } else {
            sb.append(encryption(url).toLowerCase());
        }

        if (TextUtils.isEmpty(stub)) {
            sb.append(NULL_MD5_STRING);
        } else {
            sb.append(stub);
        }


        if (TextUtils.isEmpty(ck)) {
            sb.append(NULL_MD5_STRING);
        } else {
            sb.append(encryption(ck).toLowerCase());
        }
        /*

        if (TextUtils.isEmpty(sessionid)) {
            sb.append(NULL_MD5_STRING);
        } else {
            sb.append(encryption(sessionid).toLowerCase());
        }*/
        int index = ck.indexOf("sessionid=");
        if(index == -1){
            sb.append(NULL_MD5_STRING);
        } else {
            String sessionid = ck.substring(index + 10);
            sb.append(encryption(sessionid).toLowerCase());
        }

        return sb.toString();
    }

    public static ArrayList<String> input(int timeMillis, byte[] inputBytes){


        ArrayList<String> byteTable = new ArrayList<>();

        for (int i = 0;i < 4;i++){
            if (inputBytes[i] < 0){
                byteTable.add(Integer.toHexString(inputBytes[i]).substring(6)) ;
            }else {
                byteTable.add(Integer.toHexString(inputBytes[i])) ;
            }
        }
        for (int i = 0;i < 4;i++){
            byteTable.add("0");
        }
        for (int i = 32;i < 36;i++){
            if (inputBytes[i] < 0){
                byteTable.add(Integer.toHexString(inputBytes[i]).substring(6)) ;
            }else {
                byteTable.add(Integer.toHexString(inputBytes[i])) ;
            }
        }
        for (int i = 0;i < 4;i++){
            byteTable.add("0");
        }
        String timeByte = Integer.toHexString(timeMillis);
        for (int i = 0;i < 4;i++){
            byteTable.add(timeByte.substring(2*i, 2*i+2));
        }
        return byteTable;
    }

    private static ArrayList<String> initialize(ArrayList<String> data){

        int hex = 0;
        byteTable2 = byteTable1.split(" ");
        for (int i = 0; i < data.size();i++){
            int hex1 = 0;
            if (i == 0){
                hex1 = Integer.valueOf(byteTable2[Integer.valueOf(byteTable2[0], 16) - 1], 16);
                byteTable2[i] = Integer.toHexString(hex1);
            }else if (i == 1){
                int temp = Integer.valueOf("D6", 16) + Integer.valueOf("28", 16);
                if (temp > 256){
                    temp -= 256;
                }
                hex1 = Integer.valueOf(byteTable2[temp - 1], 16);
                hex = temp;
                byteTable2[i] = Integer.toHexString(hex1);
            }else {
                int temp = hex + Integer.valueOf(byteTable2[i], 16);
                if (temp > 256){
                    temp -= 256;
                }
                hex1 = Integer.valueOf(byteTable2[temp - 1], 16);
                hex = temp;
                byteTable2[i] = Integer.toHexString(hex1);
            }
            if (hex1 * 2 >256){
                hex1 = hex1 * 2 - 256;
            }else{
                hex1 = hex1 * 2;
            }

            String hex2 = byteTable2[hex1 - 1];
            int result = Integer.valueOf(hex2, 16) ^ Integer.valueOf(data.get(i), 16);
            data.set(i, Integer.toHexString(result));
        }
        return data;
    }

    public static ArrayList<String> handle(ArrayList<String> data){

        for (int i = 0; i < data.size();i++){
            String byte1 = data.get(i);
            if (byte1.length() < 2){
                byte1 += "0";
            }else {
                byte1 = data.get(i).split("")[1] + data.get(i).split("")[0];
            }
            if (i < data.size() - 1){
                byte1 = Integer.toHexString(Integer.valueOf(byte1, 16) ^ Integer.valueOf(data.get(i + 1), 16));
            }else {
                byte1 = Integer.toHexString(Integer.valueOf(byte1, 16) ^ Integer.valueOf(data.get(0), 16));
            }
            int byte2 = ((Integer.valueOf(byte1, 16) & Integer.valueOf("55", 16)) * 2) | ((Integer.valueOf(byte1, 16) & Integer.valueOf("AA", 16)) / 2);
            byte2 = ((byte2 & Integer.valueOf("33", 16)) * 4) | ((byte2 & Integer.valueOf("CC", 16)) / 4);
            String byte3 = Integer.toHexString(byte2);
            if (byte3.length() > 1){
                byte3 = byte3.split("")[1] + byte3.split("")[0];
            }else {
                byte3 += "0";
            }
            int byte4 = Integer.valueOf(byte3, 16) ^ Integer.valueOf("FF", 16);
            byte4 = byte4 ^ Integer.valueOf("14", 16);
            data.set(i, Integer.toHexString(byte4));
        }
        return data;
    }

    public static String xGorgon(int timeMillis, byte[] inputBytes){

        ArrayList<String> data1 = new ArrayList<>();
        data1.add("3");
        data1.add("61");
        data1.add("41");
        data1.add("10");
        data1.add("80");
        data1.add("0");

        ArrayList<String> data2 = input(timeMillis, inputBytes);
        data2 = initialize(data2);
        data2 = handle(data2);
        data1.addAll(data2);

        String xGorgonStr = "";
        for (int i = 0;i < data1.size();i++){
            String temp = String.valueOf(data1.get(i));
            if (temp.length() > 1){
                xGorgonStr += temp;
            }else {
                xGorgonStr += "0";
                xGorgonStr += temp;
            }
        }

        return xGorgonStr;
    }

}

调用方式:

package com.example.douyindemo.controller;

import com.alibaba.fastjson.JSONObject;
import com.example.douyindemo.enums.ResultEnum;
import com.example.douyindemo.utils.*;
import org.springframework.web.bind.annotation.*;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

@RestController
public class DouyinController {

    @GetMapping("/feed")
    @ResponseBody
    public Result feed(@RequestParam(name = "iid")String iid, @RequestParam(name = "device_id")String device_id, @RequestParam(name = "openudid")String openudid) throws IOException {
    //public Result feed(){
        int ts = (int) (System.currentTimeMillis() / 1000);
        String _rticket = System.currentTimeMillis() + "";
        String timestamp=ts+"";
        //String iid= "84579705899";
        //String device_id= "69367187550";
        //String openudid= "e4680b0d0446ad09";
        String getParams=ParamsUtils.getFeedParams(iid,device_id,openudid,_rticket,timestamp);

        //String url = "https://aweme.snssdk.com/aweme/v1/feed/?type=0&max_cursor=0&min_cursor=0&count=6&volume=0.8666666666666667&pull_type=2&need_relieve_aweme=0&filter_warn=0&req_from=&is_cold_start=0&iid=84579705899&device_id=69367187550&ac=wifi&channel=douyin_lite_gw&aid=2329&app_name=douyin_lite&version_code=180&version_name=1.8.0&device_platform=android&ssmix=a&device_type=Redmi+Note+7+Pro&device_brand=Xiaomi&language=zh&os_api=28&os_version=9&openudid=e4680b0d0446ad09&manifest_version_code=180&resolution=1080*2119&dpi=440&update_version_code=1800&_rticket=" + _rticket + "&ts=" + ts + "&js_sdk_version=1.10.4&as=a1iosdfgh&cp=androide1";
        //String cookies = "sid_tt=74c52723180b10978d083d14fced36f2; uid_tt=593003c106c6de4a1198fb10ff9053f4; ttreq=1$85338baf79830fee82d11295117fc897e8d76892; sid_guard=74c52723180b10978d083d14fced36f2%7C1548476024%7C5184000%7CWed%2C+27-Mar-2019+04%3A13%3A44+GMT; odin_tt=7f57a517a48a1d2ad03e716ced77e48c8b03b7aa786c21e21a8a4ab7b4aae1521cf947f5fa106b969fd94ffc6b6003a372470eef06deeb01a8e550e0fc5689aa; install_id=50651589426; sessionid=74c52723180b10978d083d14fced36f2";
        String cookies = "";
        String url="https://aweme.snssdk.com/aweme/v1/feed/?"+getParams;


        String params = url.substring(url.indexOf("?") + 1, url.length());
        //String STUB = XGorgon.encryption(k).toUpperCase();
        String STUB = "";
        String s = XGorgonSell.getXGon(params, STUB, cookies);
        String Gorgon = XGorgonSell.xGorgon(ts, XGorgonSell.StrToByte(s));
        Map<String, Object> headers = new HashMap<>();
        headers.put("X-Gorgon", Gorgon);
        headers.put("X-Khronos", ts);
        headers.put("sdk-version", "1");
        headers.put("Accept-Encoding", "gzip");
        headers.put("X-SS-REQ-TICKET", _rticket);

        headers.put("User-Agent", "com.ss.android.ugc.aweme/700 (Linux; U; Android 8.0.0; zh_CN; MIX 2; Build/NRD90M; Cronet/58.0.2991.0)");
        //headers.put("x-Tt-Token","0088c943da934f04136d49c7174f68c105a7894bb5bbd56bc7ffbb4048b8a8ea54a97a7de852c031b6bae3cb8fdfeeee031e");
        headers.put("Host", "aweme.snssdk.com");
        headers.put("Cookie", cookies);
        headers.put("Connection", "Keep-Alive");
        //headers.put("x-tt-trace-id","00-cf256d8c4be04cae9a29611e42e91775-cf256d8c4be04cae-01");
        //headers.put("X-SS-STUB","30F9EAC5E02BC46BF2F5B6E046A889DD");
        String json = HttpClientUtil.doGet(url, headers, null, "UTF-8", null);
        JSONObject jsonObject = JSONObject.parseObject(json);
        if ("2154".equals(jsonObject.getString("status_code"))) {
            return ResultUtils.error(ResultEnum.NOT_FOUND_DATA.getCode(), ResultEnum.NOT_FOUND_DATA.getMsg());
        }
        return ResultUtils.success(jsonObject.getJSONArray("aweme_list"), ResultEnum.SUCCESS.getCode(), ResultEnum.SUCCESS.getMsg());
    }


    @GetMapping("/user_info")
    @ResponseBody
    public String test() {




        return null;
    }


    @GetMapping("/device_id")
    @ResponseBody
    public Result device_id() {
        String urlParams = ParamsUtils.getRegisterParams();
        String url = "https://log.snssdk.com/service/2/device_register/?" + urlParams;
        int ts = (int) (System.currentTimeMillis() / 1000);
        String params = url.substring(url.indexOf("?") + 1);
        String STUB = "";
        //String STUB =XGorgonSell.encryption(params).toUpperCase();计算出来
        String cookies = "";
        String s = XGorgonSell.getXGon(params, STUB, cookies);
        String Gorgon = XGorgonSell.xGorgon(ts, XGorgonSell.StrToByte(s));
        return null;
    }


}