Commit 2b6377fb authored by ascrutae's avatar ascrutae
Browse files

1. 增加MapReduce的java_opts的配置，在有些情况下会导致内存溢出，无法进行MapReduce 

2. 没有EntranceNode不进行Reduce
parent e9a1b4cf

skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/AnalysisServerDriver.java

+3 −0
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ import com.ai.cloud.skywalking.analysis.config.HBaseTableMetaData; 


import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.conf.Configured;

import org.apache.hadoop.hbase.HConstants;

import org.apache.hadoop.hbase.client.Scan;

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;

import org.apache.hadoop.io.Text;
@@ -55,6 +56,8 @@ public class AnalysisServerDriver extends Configured implements Tool { 
        conf.set("skywalking.analysis.mode", String.valueOf(Config.AnalysisServer.IS_ACCUMULATE_MODE));

        conf.set("hbase.zookeeper.quorum", Config.HBase.ZK_QUORUM);

        conf.set("hbase.zookeeper.property.clientPort", Config.HBase.ZK_CLIENT_PORT);

        //-XX:+UseParallelGC -XX:ParallelGCThreads=4 -XX:GCTimeRatio=10 -XX:YoungGenerationSizeIncrement=20 -XX:TenuredGenerationSizeIncrement=20 -XX:AdaptiveSizeDecrementScaleFactor=2

        conf.set("mapred.child.java.opts",Config.MapReduce.java_opts);

        String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

        if (otherArgs.length != 2) {

            System.err.println("Usage: AnalysisServer yyyy-MM-dd/HH:mm:ss yyyy-MM-dd/HH:mm:ss");

skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/chainbuild/ChainBuildMapper.java

+4 −3
Original line number Diff line number Diff line
@@ -19,7 +19,6 @@ import org.apache.hadoop.io.Text; 
import org.apache.logging.log4j.LogManager;

import org.apache.logging.log4j.Logger;





import java.io.IOException;

import java.util.*;
@@ -56,8 +55,10 @@ public class ChainBuildMapper extends TableMapper<Text, Text> { 
            chainInfo = spanToChainInfo(Bytes.toString(key.get()), spanList);

            logger.debug("convert tid[" + Bytes.toString(key.get())

                    + "] to chain with cid[" + chainInfo.getCID() + "].");

            if (chainInfo.getCallEntrance() != null && chainInfo.getCallEntrance().length() > 0) {

                context.write(

                        new Text(chainInfo.getCallEntrance()), new Text(new Gson().toJson(chainInfo)));

            }

        } catch (Exception e) {

            logger.error("Failed to mapper call chain[" + key.toString() + "]",

                    e);

skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/config/Config.java

+4 −0
Original line number Diff line number Diff line
@@ -29,4 +29,8 @@ public class Config { 
    public static class AnalysisServer{

        public static boolean IS_ACCUMULATE_MODE = true;

    }



    public static class MapReduce{

        public static String java_opts = "-Xmx200m";

    }

}

skywalking-analysis/src/main/resources/analysis.conf

+3 −1
Original line number Diff line number Diff line
@@ -16,3 +16,5 @@ filter.filter_package_name=com.ai.cloud.skywalking.analysis.chainbuild.filter.im 
chainnodesummary.interval=1



reducer.reducer_number=4



mapreduce.java_opts=-Xmx512m
 
 No newline at end of file