遗留问题整理

Impala

impala 在查询大文件的时候会直接报错：
https://issues.apache.org/jira/browse/IMPALA-1619
Support single allocations larger 1GB
版本：impala 2.3.0-cdh5.5.1

Sqoop 抽数进程僵死

sqoop 抽数进程僵死，查看jdk源码，发现是在一个函数调用的地方，尝试用Btrace,或Greys进行DEBUG，进程直接异常退出。

// 20181205

jstack -l 74372 |grep -A 50 -E 0x125b7
"main" #1 prio=5 os_prio=0 tid=0x00007f7aac019000 nid=0x125b7 runnable [0x00007f7ab1498000]
   java.lang.Thread.State: RUNNABLE
	at java.util.ComparableTimSort.mergeAt(ComparableTimSort.java:481)
	at java.util.ComparableTimSort.mergeCollapse(ComparableTimSort.java:406)
	at java.util.ComparableTimSort.sort(ComparableTimSort.java:213)
	at java.util.Arrays.sort(Arrays.java:1246)
	at com.sun.tools.javac.file.ZipFileIndex$ZipDirectory.buildIndex(ZipFileIndex.java:590)
	at com.sun.tools.javac.file.ZipFileIndex$ZipDirectory.access$000(ZipFileIndex.java:483)
	at com.sun.tools.javac.file.ZipFileIndex.checkIndex(ZipFileIndex.java:191)
	at com.sun.tools.javac.file.ZipFileIndex.<init>(ZipFileIndex.java:136)
	at com.sun.tools.javac.file.ZipFileIndexCache.getZipFileIndex(ZipFileIndexCache.java:100)
	- locked <0x00000000e379ab30> (a com.sun.tools.javac.file.ZipFileIndexCache)
	at com.sun.tools.javac.file.JavacFileManager.openArchive(JavacFileManager.java:529)
	at com.sun.tools.javac.file.JavacFileManager.openArchive(JavacFileManager.java:462)
	at com.sun.tools.javac.file.JavacFileManager.listContainer(JavacFileManager.java:348)
	at com.sun.tools.javac.file.JavacFileManager.list(JavacFileManager.java:624)
	at com.sun.tools.javac.jvm.ClassReader.fillIn(ClassReader.java:2803)
	at com.sun.tools.javac.jvm.ClassReader.complete(ClassReader.java:2446)
	at com.sun.tools.javac.jvm.ClassReader.access$000(ClassReader.java:76)
	at com.sun.tools.javac.jvm.ClassReader$1.complete(ClassReader.java:240)
	at com.sun.tools.javac.code.Symbol.complete(Symbol.java:574)
	at com.sun.tools.javac.comp.Enter.visitTopLevel(Enter.java:300)
	at com.sun.tools.javac.tree.JCTree$JCCompilationUnit.accept(JCTree.java:518)
	at com.sun.tools.javac.comp.Enter.classEnter(Enter.java:258)
	at com.sun.tools.javac.comp.Enter.classEnter(Enter.java:272)
	at com.sun.tools.javac.comp.Enter.complete(Enter.java:486)
	at com.sun.tools.javac.comp.Enter.main(Enter.java:471)
	at com.sun.tools.javac.main.JavaCompiler.enterTrees(JavaCompiler.java:982)
	at com.sun.tools.javac.main.JavaCompiler.compile(JavaCompiler.java:857)
	at com.sun.tools.javac.main.Main.compile(Main.java:523)
	at com.sun.tools.javac.api.JavacTaskImpl.doCall(JavacTaskImpl.java:129)
	at com.sun.tools.javac.api.JavacTaskImpl.call(JavacTaskImpl.java:138)
	at org.apache.sqoop.orm.CompilationManager.compile(CompilationManager.java:215)
	at org.apache.sqoop.tool.CodeGenTool.generateORM(CodeGenTool.java:108)
	at org.apache.sqoop.tool.ImportTool.importTable(ImportTool.java:478)
	at org.apache.sqoop.tool.ImportTool.run(ImportTool.java:605)
	at org.apache.sqoop.Sqoop.run(Sqoop.java:143)
	at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
	at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:179)
	at org.apache.sqoop.Sqoop.runTool(Sqoop.java:218)
	at org.apache.sqoop.Sqoop.runTool(Sqoop.java:227)
	at org.apache.sqoop.Sqoop.main(Sqoop.java:236)

   Locked ownable synchronizers:
	- None

JDK源码

    @SuppressWarnings("unchecked")
    private void mergeAt(int i) {
        assert stackSize >= 2;
        assert i >= 0;
        assert i == stackSize - 2 || i == stackSize - 3;

        int base1 = runBase[i];
        int len1 = runLen[i];
        int base2 = runBase[i + 1]; 
        int len2 = runLen[i + 1]; 
        assert len1 > 0 && len2 > 0;
        assert base1 + len1 == base2;

        /*  
         * Record the length of the combined runs; if i is the 3rd-last
         * run now, also slide over the last run (which isn't involved
         * in this merge).  The current run (i+1) goes away in any case.
         */
        runLen[i] = len1 + len2;
        if (i == stackSize - 3) {
            runBase[i + 1] = runBase[i + 2]; 
            runLen[i + 1] = runLen[i + 2]; 
        }   
        stackSize--;

        /*  
         * Find where the first element of run2 goes in run1. Prior elements
         * in run1 can be ignored (because they're already in place).
         */
        int k = gallopRight((Comparable<Object>) a[base2], a, base1, len1, 0); 
        assert k >= 0;
        base1 += k;
        len1 -= k;
        if (len1 == 0)
            return;

         /*
         * Find where the last element of run1 goes in run2. Subsequent elements
         * in run2 can be ignored (because they're already in place).
         */
        len2 = gallopLeft((Comparable<Object>) a[base1 + len1 - 1], a,
                base2, len2, len2 - 1);
        assert len2 >= 0;
        if (len2 == 0)
            return;

        // Merge remaining runs, using tmp array with min(len1, len2) elements
        if (len1 <= len2)
            mergeLo(base1, len1, base2, len2);
        else
            mergeHi(base1, len1, base2, len2);
    }  

// 483             mergeHi(base1, len1, base2, len2);         

进程Stack

"main" #1 prio=5 os_prio=0 tid=0x00007fbb5c01c800 nid=0x1ac14 runnable [0x00007fbb62713000]
   java.lang.Thread.State: RUNNABLE
        at java.util.ComparableTimSort.mergeAt(ComparableTimSort.java:483)
        at java.util.ComparableTimSort.mergeCollapse(ComparableTimSort.java:406)
        at java.util.ComparableTimSort.sort(ComparableTimSort.java:213)
        at java.util.Arrays.sort(Arrays.java:1246)
        at com.sun.tools.javac.file.ZipFileIndex$ZipDirectory.buildIndex(ZipFileIndex.java:590)
        at com.sun.tools.javac.file.ZipFileIndex$ZipDirectory.access$000(ZipFileIndex.java:483)
        at com.sun.tools.javac.file.ZipFileIndex.checkIndex(ZipFileIndex.java:191)
        at com.sun.tools.javac.file.ZipFileIndex.<init>(ZipFileIndex.java:136)
        at com.sun.tools.javac.file.ZipFileIndexCache.getZipFileIndex(ZipFileIndexCache.java:100)
        - locked <0x000000009051fab8> (a com.sun.tools.javac.file.ZipFileIndexCache)
        at com.sun.tools.javac.file.JavacFileManager.openArchive(JavacFileManager.java:529)
        at com.sun.tools.javac.file.JavacFileManager.openArchive(JavacFileManager.java:462)
        at com.sun.tools.javac.file.JavacFileManager.listContainer(JavacFileManager.java:348)
        at com.sun.tools.javac.file.JavacFileManager.list(JavacFileManager.java:624)
        at com.sun.tools.javac.jvm.ClassReader.fillIn(ClassReader.java:2803)
        at com.sun.tools.javac.jvm.ClassReader.complete(ClassReader.java:2446)
        at com.sun.tools.javac.jvm.ClassReader.access$000(ClassReader.java:76)
        at com.sun.tools.javac.jvm.ClassReader$1.complete(ClassReader.java:240)
        at com.sun.tools.javac.code.Symbol.complete(Symbol.java:574)
        at com.sun.tools.javac.comp.Enter.visitTopLevel(Enter.java:300)
        at com.sun.tools.javac.tree.JCTree$JCCompilationUnit.accept(JCTree.java:518)
        at com.sun.tools.javac.comp.Enter.classEnter(Enter.java:258)
        at com.sun.tools.javac.comp.Enter.classEnter(Enter.java:272)
        at com.sun.tools.javac.comp.Enter.complete(Enter.java:486)
        at com.sun.tools.javac.comp.Enter.main(Enter.java:471)
        at com.sun.tools.javac.main.JavaCompiler.enterTrees(JavaCompiler.java:982)
        at com.sun.tools.javac.main.JavaCompiler.compile(JavaCompiler.java:857)
        at com.sun.tools.javac.main.Main.compile(Main.java:523)
        at com.sun.tools.javac.api.JavacTaskImpl.doCall(JavacTaskImpl.java:129)
        at com.sun.tools.javac.api.JavacTaskImpl.call(JavacTaskImpl.java:138)
        at org.apache.sqoop.orm.CompilationManager.compile(CompilationManager.java:215)
        at org.apache.sqoop.tool.CodeGenTool.generateORM(CodeGenTool.java:108)
        at org.apache.sqoop.tool.ImportTool.importTable(ImportTool.java:478)
        at org.apache.sqoop.tool.ImportTool.run(ImportTool.java:605)
        at org.apache.sqoop.Sqoop.run(Sqoop.java:143)
        at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
        at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:179)
        at org.apache.sqoop.Sqoop.runTool(Sqoop.java:218)
        at org.apache.sqoop.Sqoop.runTool(Sqoop.java:227)
        at org.apache.sqoop.Sqoop.main(Sqoop.java:236)