Kafka2.1.1编译

Kafka2.1.1编译

一、下载

1.kafka源码
http://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.1.1/kafka-2.1.1-src.tgz

2.解压下载的kafka源码，比如解压后的目录为/root/kafka/kafka-2.1.1-src

3.下载scala2.11.11和gradle-4.10.2并配置环境变量

二、修改build.gradle

修改/root/kafka/kafka-2.1.1-src目录下的build.gradle文件

import org.ajoberstar.grgit.Grgit

ScalaCompileOptions.metaClass.daemonServer = true//增加
ScalaCompileOptions.metaClass.fork = true//增加
ScalaCompileOptions.metaClass.useAnt = false//增加
ScalaCompileOptions.metaClass.useCompileDaemon = false//增加

buildscript {
  repositories {
    mavenLocal()//修改
    maven {
      url "仓库1"//修改
    }
    maven {
      url "仓库2"//修改
    }
    maven {
      url "https://maven.aliyun.com/repository/gradle-plugin"//修改
    }
    //jcenter() 注释
  }
  apply from: file('gradle/buildscript.gradle'), to: buildscript

  dependencies {
    // For Apache Rat plugin to ignore non-Git files
    classpath "org.ajoberstar:grgit:1.9.3"
    classpath 'com.github.ben-manes:gradle-versions-plugin:0.20.0'
    classpath 'org.scoverage:gradle-scoverage:2.5.0'
    classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.2'
    classpath 'org.owasp:dependency-check-gradle:3.3.4'
    classpath "com.diffplug.spotless:spotless-plugin-gradle:3.16.0"
    classpath "com.github.spotbugs:spotbugs-gradle-plugin:1.6.5"//修改
  }
...
apply from: "$rootDir/gradle/dependencies.gradle"

allprojects {

  repositories {
    mavenLocal()//修改
    maven {
      url "仓库1"//修改
    }
    maven {
      url "仓库2"//修改
    }
  }

三、编译

1. 进入/root/kafka/kafka-2.1.1-src目录
2. 执行 gradle命令，上述修改了spotbugs-gradle-plugin路径是由于在jcenter里下载不到，所以改成到https://maven.aliyun.com/repository/gradle-plugin中下载，并修改插件路径
3. 第二步执行成功后会在当前目录生成gradlew
4. 执行 ./gradlew clean
5. 执行 ./gradlew -PscalaVersion=2.11 releaseTarGz

当执行gradlew时会去下载gradle-4.10.3-bin.zip，我在执行的时候报了unable to find valid certification path to requested target的错误，有两个解决办法：

1. 手动下载gradle-4.10.3-bin.zip放到/root/.gradle/wrapper/dists/gradle-4.10.3-bin/31t79e2qsceia4mkbojplrgx，这里的目录根据需要的gradle版本来确定。我这里是4.10.3。

2. 利用如下代码生成证书：

import java.io.*;
import java.net.URL;

import java.security.*;
import java.security.cert.*;

import javax.net.ssl.*;

public class InstallCert {

    // 我们要访问的HTTPS服务，如访问 https://www.shanhy.com
    public static final String hostName = "services.gradle.org";

    public static void main(String[] args) throws Exception {
        args = new String[]{hostName};
        String host;
        int port;
        char[] passphrase;
        if ((args.length == 1) || (args.length == 2)) {
            String[] c = args[0].split(":");
            host = c[0];
            port = (c.length == 1) ? 443 : Integer.parseInt(c[1]);
            String p = (args.length == 1) ? "changeit" : args[1];
            passphrase = p.toCharArray();
        } else {
            System.out.println("Usage: java InstallCert <host>[:port] [passphrase]");
            return;
        }

        File file = new File("jssecacerts");
        if (file.isFile() == false) {
            char SEP = File.separatorChar;
            File dir = new File(System.getProperty("java.home") + SEP + "lib" + SEP + "security");
            file = new File(dir, "jssecacerts");
            if (file.isFile() == false) {
                file = new File(dir, "cacerts");
            }
        }
        System.out.println("Loading KeyStore " + file + "...");
        InputStream in = new FileInputStream(file);
        KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
        ks.load(in, passphrase);
        in.close();

        SSLContext context = SSLContext.getInstance("TLS");
        TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
        tmf.init(ks);
        X509TrustManager defaultTrustManager = (X509TrustManager) tmf.getTrustManagers()[0];
        SavingTrustManager tm = new SavingTrustManager(defaultTrustManager);
        context.init(null, new TrustManager[] { tm }, null);
        SSLSocketFactory factory = context.getSocketFactory();

        System.out.println("Opening connection to " + host + ":" + port + "...");
        SSLSocket socket = (SSLSocket) factory.createSocket(host, port);
        socket.setSoTimeout(10000);
        try {
            System.out.println("Starting SSL handshake...");
            socket.startHandshake();
            socket.close();
            System.out.println();
            System.out.println("No errors, certificate is already trusted");
        } catch (SSLException e) {
            System.out.println();
            e.printStackTrace(System.out);
        }

        X509Certificate[] chain = tm.chain;
        if (chain == null) {
            System.out.println("Could not obtain server certificate chain");
            return;
        }

        BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));

        System.out.println();
        System.out.println("Server sent " + chain.length + " certificate(s):");
        System.out.println();
        MessageDigest sha1 = MessageDigest.getInstance("SHA1");
        MessageDigest md5 = MessageDigest.getInstance("MD5");
        for (int i = 0; i < chain.length; i++) {
            X509Certificate cert = chain[i];
            System.out.println(" " + (i + 1) + " Subject " + cert.getSubjectDN());
            System.out.println("   Issuer  " + cert.getIssuerDN());
            sha1.update(cert.getEncoded());
            System.out.println("   sha1    " + toHexString(sha1.digest()));
            md5.update(cert.getEncoded());
            System.out.println("   md5     " + toHexString(md5.digest()));
            System.out.println();
        }

        System.out.println("Enter certificate to add to trusted keystore or 'q' to quit: [1]");
        String line = reader.readLine().trim();
        int k;
        try {
            k = (line.length() == 0) ? 0 : Integer.parseInt(line) - 1;
        } catch (NumberFormatException e) {
            System.out.println("KeyStore not changed");
            return;
        }

        X509Certificate cert = chain[k];
        String alias = host + "-" + (k + 1);
        ks.setCertificateEntry(alias, cert);

        OutputStream out = new FileOutputStream("jssecacerts");
        ks.store(out, passphrase);
        out.close();

        System.out.println();
        System.out.println(cert);
        System.out.println();
        System.out.println("Added certificate to keystore 'jssecacerts' using alias '" + alias + "'");
    }

    private static final char[] HEXDIGITS = "0123456789abcdef".toCharArray();

    private static String toHexString(byte[] bytes) {
        StringBuilder sb = new StringBuilder(bytes.length * 3);
        for (int b : bytes) {
            b &= 0xff;
            sb.append(HEXDIGITS[b >> 4]);
            sb.append(HEXDIGITS[b & 15]);
            sb.append(' ');
        }
        return sb.toString();
    }

    private static class SavingTrustManager implements X509TrustManager {

        private final X509TrustManager tm;
        private X509Certificate[] chain;

        SavingTrustManager(X509TrustManager tm) {
            this.tm = tm;
        }

        public X509Certificate[] getAcceptedIssuers() {
            throw new UnsupportedOperationException();
        }

        public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
            throw new UnsupportedOperationException();
        }

        public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
            this.chain = chain;
            tm.checkServerTrusted(chain, authType);
        }
    }

}

将包含以上代码的java文件放到服务器上编译执行，按1回车之后会在java文件所在目录生成证书jssecacerts，将此文件拷贝到/opt/jdk/jdk8u222-b10/jre/lib/security下即可。

再次执行不再报unable to find valid certification path to requested target这个错，但又报了gradle-4.10.3-bin.zip解压的错误，进入/root/.gradle/wrapper/dists/gradle-4.10.3-bin/31t79e2qsceia4mkbojplrgx目录，发现文件很小，应该是没下载全，于是自己去下载了gradle-4.10.3-bin.zip文件放入此目录下，下载链接为https://services.gradle.org/distributions/gradle-4.10.3-bin.zip，再次执行./gradlew clean成功

./gradlew -PscalaVersion=2.11 releaseTarGz执行成功后会在/root/kafka/kafka-2.1.1-src/core/build/distributions目录下生成kafka_2.11-2.1.1.tgz，即这个包来安装即可。
附上编译脚本：

#!/bin/bash

rm -f kafka_2.11-0.10.0.1.tar.gz
chmod +x bin/*.sh
gradle -PscalaVersion=2.11
./gradlew clean
./gradlew -PscalaVersion=2.11 releaseTarGz

if [ $? -eq 0 ]; then
    cp core/build/distributions/kafka_2.11-0.10.0.1.tgz kafka_2.11-0.10.0.1.tar.gz
    ln -sf core/build/distributions/kafka_2.11-0.10.0.1.tgz
fi

参考文献

https://blog.csdn.net/frankcheng5143/article/details/52022289