스파크는 초보적으로 wordcount부터 시작합니다.

스파크 초보. - wordcount부터.

준비 작업

from pyspark.sql import SparkSession
from operator import add

#    spark  ，       wordcount
spark = SparkSession.builder.appName("WordCount").getOrCreate()

#        
#   data dataframe  ，          
data = spark.read.text("README.md")

#        
f = data.first()
f

Row(value='# Apache Spark')

#    5   
data.take(5)

[Row(value='# Apache Spark'),
 Row(value=''),
 Row(value='Spark is a fast and general cluster computing system for Big Data. It provides'),
 Row(value='high-level APIs in Scala, Java, Python, and R, and an optimized engine that'),
 Row(value='supports general computation graphs for data analysis. It also supports a')]

#       rdd  ，    
data2 = data.rdd.map(lambda x: x[0])
#        ，       string  
data2.first()

'# Apache Spark'

#            ，      
data3 = data2.flatMap(lambda x: x.split(' '))
#    5   ，             
data3.take(5)

['#', 'Apache', 'Spark', '', 'Spark']

#          1
data4 = data3.map(lambda x: (x,1))
#    turple  ，   key，           
data4.take(5)

[('#', 1), ('Apache', 1), ('Spark', 1), ('', 1), ('Spark', 1)]

#              
data5 = data4.reduceByKey(add)
#    turple  ，          
data5.take(10)

[('#', 1),
 ('Apache', 1),
 ('Spark', 16),
 ('', 71),
 ('is', 6),
 ('a', 8),
 ('fast', 1),
 ('and', 9),
 ('general', 3),
 ('cluster', 2)]

#            
res = data5.sortBy(lambda x: x[1], ascending=False).collect()
res[:5]

[('', 71), ('the', 24), ('to', 17), ('Spark', 16), ('for', 12)]

전체 코드

from pyspark.sql import SparkSession
from operator import add

spark = SparkSession.builder.appName("WordCount").getOrCreate()
data = spark.read.text("README.md")

data1 = data.rdd.map(lambda x: x[0])

data2 = data1.flatMap(lambda x: x.split(' ')).map(lambda x: (x, 1)).reduceByKey(add)

res = data2.sortBy(lambda x: x[1], ascending=False).collect()

print(res[:10])

[('', 71), ('the', 24), ('to', 17), ('Spark', 16), ('for', 12), ('and', 9), ('##', 9), ('a', 8), ('can', 7), ('on', 7)]