Add env python for Spark

This commit is contained in:
Yûki VACHOT 2024-01-11 14:01:57 +01:00
parent 1e42b00ce7
commit e28c446569
8 changed files with 79 additions and 32 deletions

27
spark_check.py Normal file
View file

@ -0,0 +1,27 @@
from dotenv import load_dotenv
import sys
import os
from pyspark.sql import SparkSession
load_dotenv()
print(os.environ["SPARK_HOME"])
print(os.environ["HADOOP_HOME"])
print(os.environ["JAVA_HOME"])
print("EXEC:")
print(sys.executable)
spark = SparkSession.builder.getOrCreate()
df = spark.createDataFrame(
[
(1, "val1"),
(2, "val2"),
(3, "val3"),
(4, "val4"),
]
)
df.show()
df.coalesce(1).write.mode("overwrite").csv("output/testoutput")
spark.stop()
print("Done\n")