用PHP和Shell写Hadoop的MapReduce程序(2)

# 如果待处理文件数目为零, 则退出执行
log_files_amount=$(($(echo $log_files | wc -l) + 0))
if [ $log_files_amount -lt 1 ]
then
 echo "no log files found"
 exit 0
fi

# 输入文件列表
for f in $log_files
do
 input_files_list="${input_files_list} $f"
done

function map_reduce () {
 if ${hadoop_home}bin/hadoop jar ${streaming_jar_path} -input${input_files_list} -output ${mapreduce_output_dir}${date}/${1}/ -mapper "${mapper} ${1}" -reducer "${reducer}" -file "${mapper}"
 then
  echo "streaming job done!"
 else
  exit 1
 fi
}

# 循环处理每一个bucket
for bucket in ${bucket_list[@]}
do
 map_reduce $bucket
done

您可能感兴趣的文章: