Pyspark에 대한 expr 함수

개요

사용 예

예1

xy_df = xy_df.withColumn('distance',F.expr('sqrt(x * x + y * y)'))

예2

rand_df = rand_df.withColumn('random',F.expr('rand() * 2 - 1'))

예3

darts = 10000

l = []

for x in range(darts):
    l.append([x])
#   l.append((x,)) tupleの場合
# https://jamiekt.wordpress.com/2016/12/13/creating-a-spark-dataframe-containing-only-one-column/

xy_df = spark.createDataFrame(l, ["index"])

xy_df = (
    xy_df
    .withColumn("x", F.expr("rand() * 2 - 1"))
    .withColumn("y", F.expr("rand() * 2 - 1"))
    .withColumn("distance", F.expr("sqrt(x * x + y * y)"))
    .withColumn("inside_a_circle", F.expr("CASE WHEN distance < 1 THEN True ELSE False END")
    )
)

4 * xy_df.filter('inside_a_circle = True').count() / 10000