修改Recommend.java
package org.conan.myhadoop.recommend; import java.util.HashMap; import java.util.Map; import java.util.regex.Pattern; import org.apache.hadoop.mapred.JobConf; import org.conan.myhadoop.hdfs.HdfsDAO; public class Recommend { public static final String HDFS = "hdfs://192.168.1.210:9000"; public static final Pattern DELIMITER = Pattern.compile("[\t,]"); public static void main(String[] args) throws Exception { Map path = new HashMap(); path.put("data", "logfile/small.csv"); path.put("Step1Input", HDFS + "/user/hdfs/recommend"); path.put("Step1Output", path.get("Step1Input") + "/step1"); path.put("Step2Input", path.get("Step1Output")); path.put("Step2Output", path.get("Step1Input") + "/step2"); path.put("Step3Input1", path.get("Step1Output")); path.put("Step3Output1", path.get("Step1Input") + "/step3_1"); path.put("Step3Input2", path.get("Step2Output")); path.put("Step3Output2", path.get("Step1Input") + "/step3_2"); path.put("Step4Input1", path.get("Step3Output1")); path.put("Step4Input2", path.get("Step3Output2")); path.put("Step4Output", path.get("Step1Input") + "/step4"); path.put("Step5Input1", path.get("Step3Output1")); path.put("Step5Input2", path.get("Step3Output2")); path.put("Step5Output", path.get("Step1Input") + "/step5"); path.put("Step6Input", path.get("Step5Output")); path.put("Step6Output", path.get("Step1Input") + "/step6"); Step1.run(path); Step2.run(path); Step3.run1(path); Step3.run2(path); //Step4.run(path); Step4_Update.run(path); Step4_Update2.run(path); System.exit(0); } public static JobConf config() { JobConf conf = new JobConf(Recommend.class); conf.setJobName("Recommand"); conf.addResource("classpath:/hadoop/core-site.xml"); conf.addResource("classpath:/hadoop/hdfs-site.xml"); conf.addResource("classpath:/hadoop/mapred-site.xml"); conf.set("io.sort.mb", "1024"); return conf; } }运行Step4_Update.java，查看输出结果
~ hadoop fs -cat /user/hdfs/recommend/step5/part-r-00000 3 107,2.0 2 107,2.0 1 107,5.0 5 107,4.0 4 107,5.0 3 106,4.0 2 106,4.0 1 106,10.0 5 106,8.0 4 106,10.0 3 105,4.0 2 105,4.0 1 105,10.0 5 105,8.0 4 105,10.0 3 104,8.0 2 104,8.0 1 104,20.0 5 104,16.0 4 104,20.0 3 103,8.0 2 103,8.0 1 103,20.0 5 103,16.0 4 103,20.0 3 102,6.0 2 102,6.0 1 102,15.0 5 102,12.0 4 102,15.0 3 101,10.0 2 101,10.0 1 101,25.0 5 101,20.0 4 101,25.0 2 106,2.5 1 106,3.0 5 106,3.0 2 105,2.5 1 105,3.0 5 105,3.0 2 104,5.0 1 104,6.0 5 104,6.0 2 103,7.5 1 103,9.0 5 103,9.0 2 102,7.5 1 102,9.0 5 102,9.0 2 101,7.5 1 101,9.0 5 101,9.0 2 106,10.0 1 106,5.0 5 106,4.0 4 106,6.0 2 105,5.0 1 105,2.5 5 105,2.0 4 105,3.0 2 104,15.0 1 104,7.5 5 104,6.0 4 104,9.0 2 103,20.0 1 103,10.0 5 103,8.0 4 103,12.0 2 102,15.0 1 102,7.5 5 102,6.0 4 102,9.0 2 101,20.0 1 101,10.0 5 101,8.0 4 101,12.0 3 107,4.0 2 107,2.0 5 107,4.0 4 107,4.5 3 106,8.0 2 106,4.0 5 106,8.0 4 106,9.0 3 105,8.0 2 105,4.0 5 105,8.0 4 105,9.0 3 104,16.0 2 104,8.0 5 104,16.0 4 104,18.0 3 103,12.0 2 103,6.0 5 103,12.0 4 103,13.5 3 102,8.0 2 102,4.0 5 102,8.0 4 102,9.0 3 101,16.0 2 101,8.0 5 101,16.0 4 101,18.0 3 107,4.5 5 107,3.5 3 106,4.5 5 106,3.5 3 105,9.0 5 105,7.0 3 104,9.0 5 104,7.0 3 103,4.5 5 103,3.5 3 102,4.5 5 102,3.5 3 101,9.0 5 101,7.0 5 106,8.0 4 106,8.0 5 105,4.0 4 105,4.0 5 104,8.0 4 104,8.0 5 103,8.0 4 103,8.0 5 102,4.0 4 102,4.0 5 101,8.0 4 101,8.0 3 107,5.0 3 105,5.0 3 104,5.0 3 101,5.0