Pig简单的代码实例:报表统计行业中的点击和曝

注意:pig中用run或者exec 运行脚本。除了cd和ls,其他命令不用。在本代码中用rm和mv命令做例子,容易出错。

另外,pig只有在store或dump时候才会真正加载数据,否则,只是加载代码,不具体操作数据。所以在rm操作时必须注意该文件是否已经生成。如果rm的文件为生成,可以第三文件,进行mv改名操作

SET job.name 'test_age_reporth_istorical';-- 定义任务名字,在:50030/jobtracker.jsp中查看任务状态,失败成功。

SET job.priority HIGH;--优先级

--注册jar包,用于读取sequence file和输出分析结果文件
REGISTER piggybank.jar;
DEFINE SequenceFileLoader org.apache.pig.piggybank.storage.SequenceFileLoader(); --读取二进制文件,函数名定义


%default Cleaned_Log /user/C/data/XXX/cleaned/$date/*/part* --$date是外部传入参数


%default AD_Data /user/XXX/data/xxx/metadata/ad/part*
%default Campaign_Data /user/xxx/data/xxx/metadata/campaign/part*
%default Social_Data /user/xxx/data/report/socialdata/part*


--所有的输出文件路径:
%default Industry_Path $file_path/report/historical/age/$year/industry
%default Industry_SUM $file_path/report/historical/age/$year/industry_sum
%default Industry_TMP $file_path/report/historical/age/$year/industry_tmp


%default Industry_Brand_Path $file_path/report/historical/age/$year/industry_brand
%default Industry_Brand_SUM $file_path/report/historical/age/$year/industry_brand_sum
%default Industry_Brand_TMP $file_path/report/historical/age/$year/industry_brand_tmp


%default ALL_Path $file_path/report/historical/age/$year/all
%default ALL_SUM $file_path/report/historical/age/$year/all_sum
%default ALL_TMP $file_path/report/historical/age/$year/all_tmp


%default output_path /user/xxx/tmp/result


origin_cleaned_data = LOAD '$Cleaned_Log' USING PigStorage(',') --读取日志文件
AS (ad_network_id:chararray,
    xxx_ad_id:chararray,
    guid:chararray,
    id:chararray,
    create_time:chararray,
    action_time:chararray,
    log_type:chararray,
    ad_id:chararray,
    positioning_method:chararray,
    location_accuracy:chararray,
    lat:chararray,
    lon:chararray,
    cell_id:chararray,
    lac:chararray,
    mcc:chararray,
    mnc:chararray,
    ip:chararray,
    connection_type:chararray,
    Android_id:chararray,
    android_advertising_id:chararray,
    openudid:chararray,
    mac_address:chararray,
    uid:chararray,
    density:chararray,
    screen_height:chararray,
    screen_width:chararray,
    user_agent:chararray,
    app_id:chararray,
    app_category_id:chararray,
    device_model_id:chararray,
    carrier_id:chararray,
    os_id:chararray,
    device_type:chararray,
    os_version:chararray,
    country_region_id:chararray,
    province_region_id:chararray,
    city_region_id:chararray,
    ip_lat:chararray,
    ip_lon:chararray,
    quadkey:chararray);


--loading metadata/ad(adId,campaignId)
metadata_ad = LOAD '$AD_Data' USING PigStorage(',') AS (adId:chararray, campaignId:chararray);


--loading metadata/campaign数据(campaignId, industryId, brandId)
metadata_campaign = LOAD '$Campaign_Data' USING PigStorage(',') AS (campaignId:chararray, industryId:chararray, brandId:chararray);


--ad and campaign for inner join
joinAdCampaignByCampaignId = JOIN metadata_ad BY campaignId,metadata_campaign BY campaignId;--(adId,campaignId,campaignId,industryId,brandId)
--filtering out redundant column of joinAdCampaignByCampaignId
joined_ad_campaign_data = FOREACH joinAdCampaignByCampaignId GENERATE $0 AS adId,$3 AS industryId,$4 AS brandId; --(adId,industryId,brandId)


--extract column for analyzing
origin_historical_age = FOREACH origin_cleaned_data GENERATE xxx_ad_id,guid,log_type;--(xxx_ad_id,guid,log_type)
--distinct
distinct_origin_historical_age = DISTINCT origin_historical_age;--(xxx_ad_id,guid,log_type)


--loading metadata_region(guid_social, sex, age, income, edu, hobby)
metadata_social = LOAD '$Social_Data' USING PigStorage(',') AS (guid_social:chararray, sex:chararray, age:chararray, income:chararray, edu:chararray, hobby:chararray);
--extract needed column in metadata_social
social_age = FOREACH metadata_social GENERATE guid_social,age;

内容版权声明:除非注明,否则皆为本站原创文章。

转载注明出处:https://www.heiqu.com/b8a038ef18218f7465c6baa6d19fd9d8.html