pyspark 通过list 构建rdd

​​pyspark 通过list 构建rdd​​

from pyspark import SparkContext ,SparkConf

conf=SparkConf().setAppName("miniProject").setMaster("local[4]")
#conf=SparkConf().setAppName("lg").setMaster("spark://192.168.10.182:7077")
sc = SparkContext(conf=conf)

#b=sc.parallelize([0, 2, 3, 4, 6], 5).glom().collect()

data = list(range(10**4))
distData = sc.parallelize(data)
打印rdd
print(distData.collect())

reduce=distData.reduce(lambda a, b: a + b)

print(reduce)


#rdd = sc.parallelize(range(1, 4)).map(lambda x: (x, "a" * x))
#
##rdd.saveAsSequenceFile("path/to/file")
#
#print(sorted(sc.sequenceFile("path/to/file").collect()))
sc.stop()

2019-03-07 22:14 ​​luoganttcc​​