week07_DataFrame.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "findspark.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark.sql"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder\\\n",
    "        .master('local')\\\n",
    "        .getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "menuDF = spark \\\n",
    "        .read \\\n",
    "        .format('csv') \\\n",
    "        .load('dessert_menu_w_Schema.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- _c0: string (nullable = true)\n",
      " |-- _c1: string (nullable = true)\n",
      " |-- _c2: string (nullable = true)\n",
      " |-- _c3: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "menuDF.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+-------------------+-----+----+\n",
      "|   _c0|                _c1|  _c2| _c3|\n",
      "+------+-------------------+-----+----+\n",
      "|menuId|               name|price|kcal|\n",
      "|   D-0|  Chocolate Parfeit| 4900| 420|\n",
      "|   D-1|    Pudding Parfeit| 5300| 380|\n",
      "|   D-2|Strawberry Icecream| 5200| 320|\n",
      "|   D-3|       Panna Mousse| 4200| 283|\n",
      "|   D-4|      Cheese Mousse| 5800| 288|\n",
      "|   D-5|           Affogato| 3000| 248|\n",
      "|   D-6|           Tiramisu| 6000| 251|\n",
      "|   D-7|     Greentea Latte| 4500| 380|\n",
      "|   D-8|     Vanilla Gelato| 3600| 131|\n",
      "|   D-9|    Caramel Parfeit| 3900| 440|\n",
      "|  D-10|           Cream An| 5000| 250|\n",
      "+------+-------------------+-----+----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "menuDF.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+-----------------+-----+----+\n",
      "|   _c0|              _c1|  _c2| _c3|\n",
      "+------+-----------------+-----+----+\n",
      "|menuId|             name|price|kcal|\n",
      "|   D-0|Chocolate Parfeit| 4900| 420|\n",
      "|   D-1|  Pudding Parfeit| 5300| 380|\n",
      "+------+-----------------+-----+----+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "menuDF.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# modify options for data header existing *header , data type modify *schema \n",
    "menuDF = spark \\\n",
    "        .read \\\n",
    "        .format('csv') \\\n",
    "        .option('header', 'true') \\\n",
    "        .option('inferSchema', 'true') \\\n",
    "        .load('dessert_menu_w_Schema.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+-------------------+-----+----+\n",
      "|menuId|               name|price|kcal|\n",
      "+------+-------------------+-----+----+\n",
      "|   D-0|  Chocolate Parfeit| 4900| 420|\n",
      "|   D-1|    Pudding Parfeit| 5300| 380|\n",
      "|   D-2|Strawberry Icecream| 5200| 320|\n",
      "+------+-------------------+-----+----+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "menuDF.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- menuId: string (nullable = true)\n",
      " |-- name: string (nullable = true)\n",
      " |-- price: integer (nullable = true)\n",
      " |-- kcal: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "menuDF.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}