支付系统spark生产应用最佳实践
sudo -u hdfs hadoop fs -mkdir /zhifu
sudo -u hdfs hadoop fs -put shebeitest.csv /zhifu
sudo -u hdfs hadoop fs -put zhifutest.csv /zhifu
# wc -l /....../zhifu/zhifu.txt
22697609
hadoop fs -cat hdfs://1.1.1.1:1111/zhifu/zhifu.txt |more
sudo -u hdfs spark-submit --class zhifujoin --master yarn-cluster --num-executors 6 --driver-memory 8g --executor-memory 8g --executor-cores 6 --queue default /...../zhifuJoin.jar ;
运行结果
sudo -u hdfs hadoop fs -cat /....../result.json/part-* | wc -l
119592
