Init

src/pyspark_training/output_dataset_1/remove_extra_space.py

@@ -0,0 +1,3 @@
+def remove_extra_spaces(df, column_name):
+    df_transformed = df.withColumn(column_name, F.regexp_replace(F.col(column_name), "\\s+", " "))
+    return df_transformed

src/test_pyspark_training/example_test.py

@@ -0,0 +1,3 @@
+
+
+def test_example_test():

src/test_pyspark_training/lib_test_utils.py

@@ -0,0 +1,16 @@
+
+def assert_df_equal(df1, df2):
+
+    try:
+        assert df1.schema() == df2.schema()
+    except AssertionError:
+        print('Error Schema')
+        print(df1.schema())
+        print(df1.schema())
+
+    try:
+        assert df1.equals(df2)
+    except AssertionError:
+        print('Error Schema')
+        df1.show()
+        df2.show()