Spark词频统计测试

from  pyspark import SparkConf, SparkContext

import jieba,Wordfilter,datetime,WordCloud,webbrowser

from operator import add

def splitWord(line):

    seg_list = jieba.cut(line, cut_all=True)

    return seg_list

if __name__ =="__main__":

     conf = SparkConf().setMaster("spark://server1.com:7077").setAppName("App %s"%datetime.datetime.now())

    sc = SparkContext(conf = conf)

    lines = sc.textFile("hdfs://server1.com:8020/data/txt/24.txt")

    counts = lines.flatMap(lambda x: splitWord(x)).map(lambda x: (x, 1)).reduceByKey(add)

    output = counts.collect()

    wcount = []

    for (word, count) in output:

        if len(word) > 2 and count > 10:

            if Wordfilter.wfilter(word):

                tup= (word,count)

                wcount.append(tup)

    sortCount = sorted(wcount,key=lambda wc:wc[1],reverse=True)

    file = "/home/hadoop/Documents/out/24.txt"

    wf = open(file,"w")

    for (w,c) in sortCount:

        wc = "%s = %s\n"%(w,c)

        wf.write(wc.encode("utf-8"))

        print(wc)

    wf.close()

    startTime = datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')

    html = r"/home/hadoop/Documents/out/d3cloudmaster/examples/" + str(startTime) + ".html"

    s = WordCloud.procTextFile(file)

    WordCloud.createWordCloudHTML(s, html)

    sc.stop()

    webbrowser.open(html)