{"id":15998,"date":"2025-07-30T13:11:57","date_gmt":"2025-07-30T13:11:57","guid":{"rendered":"http:\/\/shivaprogramming.com\/blog\/?p=15998"},"modified":"2025-07-31T23:31:27","modified_gmt":"2025-07-31T23:31:27","slug":"pyspark","status":"publish","type":"post","link":"http:\/\/shivaprogramming.com\/blog\/pyspark\/","title":{"rendered":"PySpark"},"content":{"rendered":"\n<p class=\"wp-block-paragraph\">Data<\/p>\n\n\n\n<figure class=\"wp-block-image size-full\"><a href=\"http:\/\/shivaprogramming.com\/blog\/wp-content\/uploads\/2025\/07\/image.png\"><img loading=\"lazy\" decoding=\"async\" width=\"775\" height=\"262\" src=\"http:\/\/shivaprogramming.com\/blog\/wp-content\/uploads\/2025\/07\/image.png\" alt=\"\" class=\"wp-image-16002\" srcset=\"http:\/\/shivaprogramming.com\/blog\/wp-content\/uploads\/2025\/07\/image.png 775w, http:\/\/shivaprogramming.com\/blog\/wp-content\/uploads\/2025\/07\/image-300x101.png 300w, http:\/\/shivaprogramming.com\/blog\/wp-content\/uploads\/2025\/07\/image-768x260.png 768w\" sizes=\"auto, (max-width: 775px) 100vw, 775px\" \/><\/a><\/figure>\n\n\n\n<h2 class=\"wp-block-heading\">Filter customer data by type VIP\/Regular<\/h2>\n\n\n\n<pre class=\"wp-block-code\"><code class=\"\">from pyspark.sql.functions import *  <br \/>customer=spark.sql(\"Select * FROM workspace.customerdata.customer\")<br \/>display(customer)<br \/>df.printSchema()<br \/>df1=df.filter(df[\"customer_type\"]==\"VIP\")<br \/>df0=customer.filter(col(\"customer_type\")==\"Regular\")<br \/>display(df1)<br \/># filter by customer_type and country<br \/>df1=customer.filter((customer.customer_type==\"VIP\") &amp; (customer.country=='USA'))<br \/># where condition<br \/>df2=customer.where((customer.customer_type==\"VIP\") &amp; (customer.country=='USA'))<br \/># or operator<br \/>df3=customer.where((customer.customer_type==\"VIP\") | (customer.country=='USA'))<br \/><\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">Add new column withColumn Function<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code class=\"\">from pyspark.sql.functions import *  <br \/>customer=spark.sql(\"Select * FROM workspace.customerdata.customer\")<br \/>customer =customer.withColumn(\"Salary\", col(\"age\")* 1000)<br \/>customer.printSchema()<br \/>display(customer)<br \/><br \/># withColumn fuction<br \/>customer =customer.withColumn(\"Seniority\", when(customer.age>50, \"Senior\").otherwise(\"Junior\"))<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\"><\/p>\n","protected":false},"excerpt":{"rendered":"<p>Data Filter customer data by type VIP\/Regular Add new column withColumn Function<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[31],"tags":[],"class_list":["post-15998","post","type-post","status-publish","format-standard","hentry","category-puppet-configuration-management-tool"],"_links":{"self":[{"href":"http:\/\/shivaprogramming.com\/blog\/wp-json\/wp\/v2\/posts\/15998","targetHints":{"allow":["GET"]}}],"collection":[{"href":"http:\/\/shivaprogramming.com\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/shivaprogramming.com\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/shivaprogramming.com\/blog\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/shivaprogramming.com\/blog\/wp-json\/wp\/v2\/comments?post=15998"}],"version-history":[{"count":7,"href":"http:\/\/shivaprogramming.com\/blog\/wp-json\/wp\/v2\/posts\/15998\/revisions"}],"predecessor-version":[{"id":16012,"href":"http:\/\/shivaprogramming.com\/blog\/wp-json\/wp\/v2\/posts\/15998\/revisions\/16012"}],"wp:attachment":[{"href":"http:\/\/shivaprogramming.com\/blog\/wp-json\/wp\/v2\/media?parent=15998"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/shivaprogramming.com\/blog\/wp-json\/wp\/v2\/categories?post=15998"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/shivaprogramming.com\/blog\/wp-json\/wp\/v2\/tags?post=15998"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}