pig简单介绍

                                                                  pig简单介绍

1.load:从文件中读取数据（）
  用法：athletes = LOAD 'hdfs://master:9000/OlympicAthletes.csv' USING org.apache.pig.piggybank.storage.CSVExcelStorage(',', 'YES_MULTILINE', 'NOCHANGE', 'SKIP_INPUT_HEADER')  AS (athlete:chararray, country:chararray, year:int, sport:chararray, gold:int, silver:int, bronze:int, total:int);
  注意：load和=之间有空格
2. dump :把操作结果读取到屏幕
   用法:  DUMP athletes
3.limit：限制前面多少条
   用法：athletes_lim = LIMIT athletes 10;
         DUMP athletes_lim;
4.GROUP BY：按照字段分组（根据国家分组）
  用法： athletes_grp_country = GROUP athletes BY country;
5.DESCRIBE(描述结构)
 用法：describe athletes_grp_country
6.FOREACH … GENERATE（统计出每个国家（country）赢了多少金牌(sum(athletes.total))）
  medal_sum = FOREACH athletes_grp_country GENERATE group AS country, SUM(athletes.total) as medal_count;
  DUMP medal_sum;
  注意：athletes_grp_country是按照国家分组后的结果
7.distinct（把重复国家去掉)
    distinct_countries = DISTINCT (FOREACH athletes GENERATE country);
    DUMP distinct_countries;
   这里或者可以写成这样：
   a =  FOREACH athletes GENERATE country;
   distinct_countries = DISTINCT a;
   DUMP distinct_countries;
8. GROUP ALL
   data_range = FOREACH (GROUP athletes ALL) GENERATE MIN(athletes.year) as min_year, MAX(athletes.year) as max_year;
   DUMP data_range;
  注意： MIN  MAX区分大小写
9. order  .... by ....排序（按照金牌数降序排序）
   ordered_medals = ORDER medal_sum BY medal_count DESC;
    ordered_medals_lim = LIMIT ordered_medals 1;
    DUMP ordered_medals_lim;   
10.filter(找出不是游泳，根据国家分组，最高金牌的国家的金牌数)
   athletes_filter = FILTER athletes by sport != 'Swimming';
   medal_sum = FOREACH (GROUP athletes_filter BY country) GENERATE group as country, SUM(athletes_filter.total_medals) as medal_count;
   ordered_medals = ORDER medal_sum BY medal_count DESC;
   ordered_medals_lim = LIMIT ordered_medals 1;
   DUMP ordered_medals_lim; 
   注意：问题：多少国家只赢一个金牌？
    deco = filter medial_sum by count_medial == 1;
    suna= foreach (group deco all) generate COUNT(deco.count_medial);
11. join(他们已经获得了在连续的奥运会奖牌数相同的运动员)
    copy复制  athletes_copy = FOREACH athletes GENERATE athlete, year as year2, total as total2;
       athletes_join = JOIN athletes BY athlete, athletes_copy BY athlete;
      athletes_join_filtered = FILTER athletes_join BY total == total2 AND year2 == year+4;     
      athletes_output = FOREACH athletes_join_filtered GENERATE athletes::athlete as athlete, total2, year, year2; 
    这里有点问题！！！！！！！！
12.split()
    SPLIT athletes INTO
            summer_olympics IF year % 4 == 0,
            winter_olympics IF year % 4 != 0;
















from pig_util import outputSchema


@outputSchema('score:int')
def calculate_score(gold, silver, bronze):
    return 3 * gold + 2 * silver + bronze




REGISTER 'olympic_udfs.py' USING streaming_python AS udf




athlete_score = FOREACH athletes GENERATE athlete, udf.calculate_score(gold_medal, silver_medal, bronze_medal) as score;
      


问题：
找出哪个国家分数最高的冬季奥运会上我们的新指标


解决方法：
新建olympic_udfs.py内容如下4行：
from pig_util import outputSchema
@outputSchema('score:int')
def calculate_score(gold, silver, bronze):
    return 3 * gold + 2 * silver + bronze






register '/usr/local/pig/contrib/piggybank/java/piggybank.jar'     
REGISTER '/root/olympic_udfs.py' USING streaming_python AS udf


athletes = LOAD 'hdfs://master:9000/OlympicAthletes.csv' USING org.apache.pig.piggybank.storage.CSVExcelStorage(',', 'YES_MULTILINE', 'NOCHANGE', 'SKIP_INPUT_HEADER')  AS (athlete:chararray, country:chararray, year:int, sport:chararray, gold:int, silver:int, bronze:int, total:int);
SPLIT athletes INTO summer_olympics IF year % 4 == 0, winter_olympics IF year % 4 != 0;


1.winter_medal_sum = FOREACH (GROUP winter_olympics BY country) GENERATE group AS country, SUM(winter_olympics.gold)   as gold_sum, SUM(winter_olympics.silver) as silver_sum, SUM(winter_olympics.bronze) as bronze_sum;
3.country_scores = FOREACH winter_medal_sum GENERATE country, udf.calculate_score(gold_sum, silver_sum, bronze_sum) as score;
4.ordered_winter_medals = ORDER country_scores BY score DESC;
5.ordered_winter_medals_lim = LIMIT ordered_winter_medals 1;
DUMP ordered_winter_medals_lim;