Week05_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init()\n",
    "\n",
    "import pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%e\n"
    }
   },
   "outputs": [],
   "source": [
    "from pyspark import SparkContext, SparkConf\n",
    "from pyspark.sql import SparkSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession \\\n",
    ".builder \\\n",
    ".master('local') \\\n",
    ".appName('Korean text file word Count program') \\\n",
    ".getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "linesDF = spark.read.format('text').load('/home/tteon/바탕화면/README.md')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(value='# Apache Spark'),\n",
       " Row(value=''),\n",
       " Row(value='Spark is a fast and general cluster computing system for Big Data. It provides')]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linesDF.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = pyspark.sql.SparkSession.builder.master('local').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = spark.sparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark import SparkContext ,SparkConf\n",
    "conf = pyspark.SparkConf().setMaster('local')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "numbers = sc.parallelize([1,2,3,4,5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numbers.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = sc.textFile('/home/tteon/바탕화면/README.md')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['# Apache Spark',\n",
       " '',\n",
       " 'Spark is a fast and general cluster computing system for Big Data. It provides']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "words2 = lines.flatMap(lambda line: line.split(\" \"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['#', 'Apache', 'Spark']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words2.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "unexpected EOF while parsing (<ipython-input-18-0b6b9fde3002>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001B[0;36m  File \u001B[0;32m\"<ipython-input-18-0b6b9fde3002>\"\u001B[0;36m, line \u001B[0;32m1\u001B[0m\n\u001B[0;31m    words = words2.flatMap(lambda line: re.split('\\\\W+', line)\u001B[0m\n\u001B[0m                                                              ^\u001B[0m\n\u001B[0;31mSyntaxError\u001B[0m\u001B[0;31m:\u001B[0m unexpected EOF while parsing\n"
     ]
    }
   ],
   "source": [
    "words = words2].flatMap(lambda line: re.split('\\\\W+', line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = lines.flatMap(lambda line: re.split(\"\\\\W+\", line)) \\\n",
    "             .filter(lambda word: len(word) >0) \\\n",
    "             .map(lambda word: word.lower()) \\\n",
    "             .map(lambda word: (word, 1)) \\\n",
    "             .reduceByKey(lambda v1, v2 : v1 + v2) \\\n",
    "             .sortBy(lambda x: x[1], False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('spark', 32), ('the', 25), ('to', 19)]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'words' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "\u001B[0;32m<ipython-input-17-e4b96f6cdb91>\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0mwords\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtake\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;36m3\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m",
      "\u001B[0;31mNameError\u001B[0m: name 'words' is not defined"
     ]
    }
   ],
   "source": [
    "words.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordsF = words.filter(lambda w: len(w) > 0 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Apache', 'Spark', 'Spark']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordsF.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordAndOne = wordsF.map(lambda x:(x,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Apache', 1), ('Spark', 1), ('Spark', 1)]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordAndOne.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordCount = wordAndOne.reduceByKey(lambda v1, v2 : v1 + v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Apache', 2), ('Spark', 18), ('is', 7)]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordCount.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "sortedWC = wordCount.sortByKey(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('your', 1), ('you', 4), ('yarn', 2)]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sortedWC.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "sortedWC2 = wordCount.sortBy(lambda x : x[1], False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('the', 24),\n",
       " ('Spark', 18),\n",
       " ('to', 17),\n",
       " ('spark', 14),\n",
       " ('run', 13),\n",
       " ('for', 12),\n",
       " ('and', 11),\n",
       " ('a', 9),\n",
       " ('http', 9),\n",
       " ('apache', 9),\n",
       " ('org', 9),\n",
       " ('is', 7),\n",
       " ('on', 7),\n",
       " ('html', 7),\n",
       " ('example', 7),\n",
       " ('can', 6),\n",
       " ('building', 6),\n",
       " ('tests', 6),\n",
       " ('Hadoop', 6),\n",
       " ('in', 5)]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sortedWC2.take(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "top20WordsAndCounts = lines.flatMap(lambda line: re.split(\"\\\\W+\", line)) \\\n",
    "             .filter(lambda word: len(word) >0) \\\n",
    "             .map(lambda word: word.lower()) \\\n",
    "             .map(lambda word: (word, 1)) \\\n",
    "             .reduceByKey(lambda v1, v2 : v1 + v2) \\\n",
    "             .sortBy(lambda x: x[1], False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('spark', 32),\n",
       " ('the', 25),\n",
       " ('to', 19),\n",
       " ('for', 15),\n",
       " ('run', 13),\n",
       " ('and', 12),\n",
       " ('apache', 11),\n",
       " ('a', 10),\n",
       " ('http', 9),\n",
       " ('org', 9),\n",
       " ('you', 8),\n",
       " ('building', 8),\n",
       " ('example', 8),\n",
       " ('is', 7),\n",
       " ('on', 7),\n",
       " ('html', 7),\n",
       " ('tests', 7),\n",
       " ('hadoop', 7),\n",
       " ('documentation', 6),\n",
       " ('can', 6)]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top20WordsAndCounts.take(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}