Logs.json

{"paragraphs":[{"text":"%md\n## Spark 1.6.x Demo\n##### Spark All The Transformations\n#####\n##### by Tim Spann (Shift-Enter to Run)","dateUpdated":"Apr 18, 2016 2:45:33 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461005056128_783046361","id":"20160418-144416_682654065","result":{"code":"SUCCESS","type":"HTML","msg":"<h2>Spark 1.6.x Demo</h2>\n<h5>Spark All The Transformations</h5>\n<h5></h5>\n<h5>by Tim Spann (Shift-Enter to Run)</h5>\n"},"dateCreated":"Apr 18, 2016 2:44:16 PM","dateStarted":"Apr 18, 2016 2:45:33 PM","dateFinished":"Apr 18, 2016 2:45:34 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:23"},{"text":"val originalRDD = sc.parallelize(List(\"Data1\", \"Data2\", \"Data4\", \"Data5\", \"Data6\", \"Bacon\", \"Hortonworks\", \"Hadoop\", \n\"Spark\", \"Tungesten\",\n\"SQL\"), 4)\n","dateUpdated":"Apr 18, 2016 3:35:20 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461005133748_-1892210144","id":"20160418-144533_494225985","result":{"code":"SUCCESS","type":"TEXT","msg":"originalRDD: org.apache.spark.rdd.RDD[String] = ParallelCollectionRDD[15] at parallelize at <console>:29\n"},"dateCreated":"Apr 18, 2016 2:45:33 PM","dateStarted":"Apr 18, 2016 3:34:16 PM","dateFinished":"Apr 18, 2016 3:34:17 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:24"},{"text":"val flatRDD = originalRDD.flatMap(_.split(\" \"))\n","dateUpdated":"Apr 18, 2016 3:34:32 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461005219278_1591245101","id":"20160418-144659_1048250632","result":{"code":"SUCCESS","type":"TEXT","msg":"flatRDD: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[16] at flatMap at <console>:31\n"},"dateCreated":"Apr 18, 2016 2:46:59 PM","dateStarted":"Apr 18, 2016 3:34:32 PM","dateFinished":"Apr 18, 2016 3:34:32 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:25"},{"text":"originalRDD.collect().foreach(println)","dateUpdated":"Apr 18, 2016 3:34:39 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461005691413_-424396379","id":"20160418-145451_2059858231","result":{"code":"SUCCESS","type":"TEXT","msg":"Data1\nData2\nData4\nData5\nData6\nBacon\nHortonworks\nHadoop\nSpark\nTungesten\nSQL\n"},"dateCreated":"Apr 18, 2016 2:54:51 PM","dateStarted":"Apr 18, 2016 3:34:39 PM","dateFinished":"Apr 18, 2016 3:34:40 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:26"},{"text":"originalRDD.count()","dateUpdated":"Apr 18, 2016 3:34:45 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461005719690_-584220414","id":"20160418-145519_511821456","result":{"code":"SUCCESS","type":"TEXT","msg":"res61: Long = 11\n"},"dateCreated":"Apr 18, 2016 2:55:19 PM","dateStarted":"Apr 18, 2016 3:34:45 PM","dateFinished":"Apr 18, 2016 3:34:46 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:27"},{"text":"originalRDD.first()","dateUpdated":"Apr 18, 2016 3:34:51 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461005780147_394504167","id":"20160418-145620_1916950906","result":{"code":"SUCCESS","type":"TEXT","msg":"res63: String = Data1\n"},"dateCreated":"Apr 18, 2016 2:56:20 PM","dateStarted":"Apr 18, 2016 3:34:51 PM","dateFinished":"Apr 18, 2016 3:34:51 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:28"},{"text":"originalRDD.take(2)","dateUpdated":"Apr 18, 2016 3:34:54 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461005786922_-1693158690","id":"20160418-145626_607996804","result":{"code":"SUCCESS","type":"TEXT","msg":"res65: Array[String] = Array(Data1, Data2)\n"},"dateCreated":"Apr 18, 2016 2:56:26 PM","dateStarted":"Apr 18, 2016 3:34:54 PM","dateFinished":"Apr 18, 2016 3:34:55 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:29"},{"text":"originalRDD.takeSample(true,5,7634184)","dateUpdated":"Apr 18, 2016 3:34:58 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461005794773_-309586321","id":"20160418-145634_1906271045","result":{"code":"SUCCESS","type":"TEXT","msg":"res67: Array[String] = Array(Data4, Data6, Hadoop, SQL, Bacon)\n"},"dateCreated":"Apr 18, 2016 2:56:34 PM","dateStarted":"Apr 18, 2016 3:34:58 PM","dateFinished":"Apr 18, 2016 3:34:59 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:30"},{"text":"originalRDD.takeOrdered(5)","dateUpdated":"Apr 18, 2016 3:35:03 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461005823928_-1793901605","id":"20160418-145703_32392500","result":{"code":"SUCCESS","type":"TEXT","msg":"res69: Array[String] = Array(Bacon, Data1, Data2, Data4, Data5)\n"},"dateCreated":"Apr 18, 2016 2:57:03 PM","dateStarted":"Apr 18, 2016 3:35:03 PM","dateFinished":"Apr 18, 2016 3:35:04 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:31"},{"text":"val mapped =   originalRDD.mapPartitionsWithIndex{\n                       (index, iterator) => {\n                           println(\"Index -> \" + index)\n                          val myList = iterator.toList\n                           myList.map(x => x + \" -> \" + index).iterator\n                        }\n                     }","dateUpdated":"Apr 18, 2016 3:35:11 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461005903285_-508181339","id":"20160418-145823_1706945014","result":{"code":"SUCCESS","type":"TEXT","msg":"mapped: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[19] at mapPartitionsWithIndex at <console>:33\n"},"dateCreated":"Apr 18, 2016 2:58:23 PM","dateStarted":"Apr 18, 2016 3:35:11 PM","dateFinished":"Apr 18, 2016 3:35:12 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:32"},{"text":"mapped.take(5)","dateUpdated":"Apr 18, 2016 3:35:14 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461007069011_1787890062","id":"20160418-151749_531996029","result":{"code":"SUCCESS","type":"TEXT","msg":"res72: Array[String] = Array(Data1 -> 0, Data2 -> 0, Data4 -> 1, Data5 -> 1, Data6 -> 1)\n"},"dateCreated":"Apr 18, 2016 3:17:49 PM","dateStarted":"Apr 18, 2016 3:35:14 PM","dateFinished":"Apr 18, 2016 3:35:15 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:33"},{"text":"val rddSpark = sc.parallelize(List(\"SQL\",\"Streaming\",\"GraphX\", \"MLLib\", \"Bagel\", \n\"SparkR\",\"Python\",\"Scala\",\"Java\", \"Alluxio\", \"Tungsten\", \"Zeppelin\"))\n\nval rddHadoop = sc.parallelize(List(\"HDFS\", \"YARN\", \"TEZ\", \"Hive\", \"HBase\", \"Pig\", \"Atlas\", \n\"Storm\", \"Accumulo\", \"Ranger\", \"Phoenix\", \"MapReduce\", \"Slider\", \"Flume\", \"Kafka\", \"Oozie\", \"Sqoop\", \"Falcon\",\n\"Knox\", \"Ambari\", \"Zookeeper\", \"Cloudbreak\", \"SQL\", \"Java\", \"Scala\", \"Python\"))","dateUpdated":"Apr 18, 2016 3:50:58 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461007544433_1168699387","id":"20160418-152544_465412727","result":{"code":"SUCCESS","type":"TEXT","msg":"rddSpark: org.apache.spark.rdd.RDD[String] = ParallelCollectionRDD[23] at parallelize at <console>:29\nrddHadoop: org.apache.spark.rdd.RDD[String] = ParallelCollectionRDD[24] at parallelize at <console>:30\n"},"dateCreated":"Apr 18, 2016 3:25:44 PM","dateStarted":"Apr 18, 2016 3:50:58 PM","dateFinished":"Apr 18, 2016 3:50:59 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:34"},{"text":"val bigDataRDD = rddHadoop.union(rddSpark)\nbigDataRDD.collect()","dateUpdated":"Apr 18, 2016 3:53:14 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461008993064_-1844229331","id":"20160418-154953_696142157","result":{"code":"SUCCESS","type":"TEXT","msg":"bigDataRDD: org.apache.spark.rdd.RDD[String] = UnionRDD[32] at union at <console>:33\nres84: Array[String] = Array(HDFS, YARN, TEZ, Hive, HBase, Pig, Atlas, Storm, Accumulo, Ranger, Phoenix, MapReduce, Slider, Flume, Kafka, Oozie, Sqoop, Falcon, Knox, Ambari, Zookeeper, Cloudbreak, SQL, Java, Scala, Python, SQL, Streaming, GraphX, MLLib, Bagel, SparkR, Python, Scala, Java, Alluxio, Tungsten, Zeppelin)\n"},"dateCreated":"Apr 18, 2016 3:49:53 PM","dateStarted":"Apr 18, 2016 3:53:14 PM","dateFinished":"Apr 18, 2016 3:53:14 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:35"},{"text":"rddHadoop.intersection(rddSpark).collect()","dateUpdated":"Apr 18, 2016 3:51:11 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461009017587_1520830521","id":"20160418-155017_152289551","result":{"code":"SUCCESS","type":"TEXT","msg":"res82: Array[String] = Array(Python, SQL, Java, Scala)\n"},"dateCreated":"Apr 18, 2016 3:50:17 PM","dateStarted":"Apr 18, 2016 3:51:11 PM","dateFinished":"Apr 18, 2016 3:51:12 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:36"},{"text":"bigDataRDD.distinct().collect()","dateUpdated":"Apr 18, 2016 3:53:44 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461009071829_-128325858","id":"20160418-155111_1961746938","result":{"code":"SUCCESS","type":"TEXT","msg":"res86: Array[String] = Array(Hive, TEZ, Storm, Slider, Falcon, HDFS, HBase, GraphX, Alluxio, Cloudbreak, MapReduce, Java, Ambari, Streaming, Sqoop, Atlas, Zookeeper, Python, Tungsten, YARN, Knox, SparkR, Ranger, Kafka, MLLib, SQL, Oozie, Scala, Pig, Phoenix, Bagel, Flume, Accumulo, Zeppelin)\n"},"dateCreated":"Apr 18, 2016 3:51:11 PM","dateStarted":"Apr 18, 2016 3:53:44 PM","dateFinished":"Apr 18, 2016 3:53:45 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:37"},{"text":"bigDataRDD.sample(true,0.25 ).collect()","dateUpdated":"Apr 18, 2016 3:59:48 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461009224554_1827599633","id":"20160418-155344_1598192104","result":{"code":"SUCCESS","type":"TEXT","msg":"res101: Array[String] = Array(Accumulo, Falcon, Cloudbreak, Java, Streaming, Python)\n"},"dateCreated":"Apr 18, 2016 3:53:44 PM","dateStarted":"Apr 18, 2016 3:59:48 PM","dateFinished":"Apr 18, 2016 3:59:49 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:38"},{"text":"bigDataRDD.count()","dateUpdated":"Apr 18, 2016 4:00:04 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461009278856_2107367923","id":"20160418-155438_955206019","result":{"code":"SUCCESS","type":"TEXT","msg":"res103: Long = 38\n"},"dateCreated":"Apr 18, 2016 3:54:38 PM","dateStarted":"Apr 18, 2016 4:00:04 PM","dateFinished":"Apr 18, 2016 4:00:04 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:39"},{"text":"","dateUpdated":"Apr 18, 2016 3:57:36 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461009417810_1978308475","id":"20160418-155657_1537025121","result":{"code":"SUCCESS","type":"TEXT","msg":"res95: Int = 9\n"},"dateCreated":"Apr 18, 2016 3:56:57 PM","dateStarted":"Apr 18, 2016 3:57:24 PM","dateFinished":"Apr 18, 2016 3:57:24 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:40"},{"text":"val keyValueRDD = sc.parallelize(Seq(\n                    (\"Bacon\", \"Awesome\"),\n                     (\"PorkRoll\", \"Great\"),\n                     (\"Tofu\", \"Bogus\")))","dateUpdated":"Apr 18, 2016 4:03:13 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461009427335_172681887","id":"20160418-155707_564420305","result":{"code":"SUCCESS","type":"TEXT","msg":"keyValueRDD: org.apache.spark.rdd.RDD[(String, String)] = ParallelCollectionRDD[40] at parallelize at <console>:29\n"},"dateCreated":"Apr 18, 2016 3:57:07 PM","dateStarted":"Apr 18, 2016 4:03:13 PM","dateFinished":"Apr 18, 2016 4:03:13 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:41"},{"text":"val groupByRDD = keyValueRDD.groupByKey()\ngroupByRDD.collect()","dateUpdated":"Apr 18, 2016 4:03:46 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461009793332_1645604657","id":"20160418-160313_1460399062","result":{"code":"SUCCESS","type":"TEXT","msg":"groupByRDD: org.apache.spark.rdd.RDD[(String, Iterable[String])] = ShuffledRDD[41] at groupByKey at <console>:31\nres106: Array[(String, Iterable[String])] = Array((PorkRoll,CompactBuffer(Great)), (Tofu,CompactBuffer(Bogus)), (Bacon,CompactBuffer(Awesome)))\n"},"dateCreated":"Apr 18, 2016 4:03:13 PM","dateStarted":"Apr 18, 2016 4:03:46 PM","dateFinished":"Apr 18, 2016 4:03:46 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:42"},{"text":"val kvRDD = sc.parallelize(Seq((1,\"Bacon\"), (1, \"Hamburger\"), (1,\"Cheeseburger\")))\nval reducedByRDD = kvRDD.reduceByKey((a, b) => a.concat(b))\nreducedByRDD.take(5)","dateUpdated":"Apr 18, 2016 4:20:20 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461009826279_1941907282","id":"20160418-160346_2110467647","result":{"code":"SUCCESS","type":"TEXT","msg":"kvRDD: org.apache.spark.rdd.RDD[(Int, String)] = ParallelCollectionRDD[65] at parallelize at <console>:29\nreducedByRDD: org.apache.spark.rdd.RDD[(Int, String)] = ShuffledRDD[66] at reduceByKey at <console>:31\nres136: Array[(Int, String)] = Array((1,BaconHamburgerCheeseburger))\n"},"dateCreated":"Apr 18, 2016 4:03:46 PM","dateStarted":"Apr 18, 2016 4:20:20 PM","dateFinished":"Apr 18, 2016 4:20:21 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:43"},{"text":"keyValueRDD.countByKey().foreach(println)","dateUpdated":"Apr 18, 2016 4:27:45 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461010291950_70565253","id":"20160418-161131_200077673","result":{"code":"SUCCESS","type":"TEXT","msg":"(PorkRoll,1)\n(Tofu,1)\n(Bacon,1)\n"},"dateCreated":"Apr 18, 2016 4:11:31 PM","dateStarted":"Apr 18, 2016 4:27:46 PM","dateFinished":"Apr 18, 2016 4:27:46 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:44"},{"text":"keyValueRDD.saveAsTextFile(\"here\")","dateUpdated":"Apr 18, 2016 4:29:18 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461011180966_820788676","id":"20160418-162620_1358469825","result":{"code":"SUCCESS","type":"TEXT","msg":""},"dateCreated":"Apr 18, 2016 4:26:20 PM","dateStarted":"Apr 18, 2016 4:29:18 PM","dateFinished":"Apr 18, 2016 4:29:19 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:45"},{"text":"keyValueRDD.saveAsSequenceFile(\"here2\")\n\nkeyValueRDD.saveAsObjectFile(\"here3\")","dateUpdated":"Apr 18, 2016 4:33:06 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461011352020_-23893858","id":"20160418-162912_1537820517","result":{"code":"SUCCESS","type":"TEXT","msg":""},"dateCreated":"Apr 18, 2016 4:29:12 PM","dateStarted":"Apr 18, 2016 4:32:04 PM","dateFinished":"Apr 18, 2016 4:32:05 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:46"},{"text":"%sh ls ","dateUpdated":"Apr 18, 2016 4:32:10 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/sh"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461011397367_-1629789795","id":"20160418-162957_1466990956","result":{"code":"SUCCESS","type":"TEXT","msg":"DISCLAIMER\nLICENSE\nNOTICE\nREADME.md\nbin\nconf\nderby.log\nhere\nhere2\nhere3\ninterpreter\nlib\nlicenses\nlogs\nmetastore_db\nnotebook\nrun\nwebapps\nzeppelin-server-0.5.6-incubating.jar\nzeppelin-web-0.5.6-incubating.war\n"},"dateCreated":"Apr 18, 2016 4:29:57 PM","dateStarted":"Apr 18, 2016 4:32:10 PM","dateFinished":"Apr 18, 2016 4:32:10 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:47"},{"text":"%sh ls here\n\ncat here/part-00003","dateUpdated":"Apr 18, 2016 4:33:17 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/sh"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461011404626_-1605150541","id":"20160418-163004_99185925","result":{"code":"SUCCESS","type":"TEXT","msg":"_SUCCESS\npart-00000\npart-00001\npart-00002\npart-00003\n(Tofu,Bogus)\n"},"dateCreated":"Apr 18, 2016 4:30:04 PM","dateStarted":"Apr 18, 2016 4:33:17 PM","dateFinished":"Apr 18, 2016 4:33:17 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:48"},{"text":"bigDataRDD.reduce((a, b) => a.concat(b))","dateUpdated":"Apr 18, 2016 4:36:02 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461011537137_-1792707145","id":"20160418-163217_1109136632","result":{"code":"SUCCESS","type":"TEXT","msg":"res154: String = AmbariZookeeperCloudbreakSQLJavaScalaPythonFlumeKafkaOozieSqoopFalconKnoxAtlasStormAccumuloRangerPhoenixMapReduceSliderHDFSYARNTEZHiveHBasePigSQLStreamingGraphXMLLibBagelSparkRPythonScalaJavaAlluxioTungstenZeppelin\n"},"dateCreated":"Apr 18, 2016 4:32:17 PM","dateStarted":"Apr 18, 2016 4:36:02 PM","dateFinished":"Apr 18, 2016 4:36:02 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:49"},{"text":"val namesRDD = sc.parallelize(List((1, 25), (1, 27), (3, 25), (3, 27)))\nval groupByRDD = namesRDD.aggregateByKey(0)((k,v) => v.toInt+k, (v,k) => k+v).collect()\n","dateUpdated":"Apr 18, 2016 4:55:03 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461011705093_350003807","id":"20160418-163505_1694063858","result":{"code":"SUCCESS","type":"TEXT","msg":"namesRDD: org.apache.spark.rdd.RDD[(Int, Int)] = ParallelCollectionRDD[92] at parallelize at <console>:29\ngroupByRDD: Array[(Int, Int)] = Array((1,52), (3,52))\n"},"dateCreated":"Apr 18, 2016 4:35:05 PM","dateStarted":"Apr 18, 2016 4:55:03 PM","dateFinished":"Apr 18, 2016 4:55:04 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:50"},{"text":"val sortByRDD = namesRDD.sortByKey(true).collect()","dateUpdated":"Apr 18, 2016 4:56:42 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461012302163_1460263251","id":"20160418-164502_1984896042","result":{"code":"SUCCESS","type":"TEXT","msg":"sortByRDD: Array[(Int, Int)] = Array((1,25), (1,27), (3,25), (3,27))\n"},"dateCreated":"Apr 18, 2016 4:45:02 PM","dateStarted":"Apr 18, 2016 4:56:42 PM","dateFinished":"Apr 18, 2016 4:56:42 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:51"},{"text":"val sortByRDD = namesRDD.sortByKey(false).collect()","dateUpdated":"Apr 18, 2016 4:57:37 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461012981654_-956023904","id":"20160418-165621_570757480","result":{"code":"SUCCESS","type":"TEXT","msg":"sortByRDD: Array[(Int, Int)] = Array((3,25), (3,27), (1,25), (1,27))\n"},"dateCreated":"Apr 18, 2016 4:56:21 PM","dateStarted":"Apr 18, 2016 4:57:37 PM","dateFinished":"Apr 18, 2016 4:57:37 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:52"},{"text":"val otherKeyValueRDD = sc.parallelize(Seq(\n                    (\"Bacon\", \"Amazing\"),\n                     (\"Steak\", \"Fine\"),\n                     (\"Lettuce\", \"Sad\")))\nkeyValueRDD.join(otherKeyValueRDD).collect()\n\nkeyValueRDD.leftOuterJoin(otherKeyValueRDD).collect()\nkeyValueRDD.rightOuterJoin(otherKeyValueRDD).collect()\nkeyValueRDD.fullOuterJoin(otherKeyValueRDD).collect()","dateUpdated":"Apr 18, 2016 5:02:45 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461013057286_-745057269","id":"20160418-165737_1188058593","result":{"code":"SUCCESS","type":"TEXT","msg":"otherKeyValueRDD: org.apache.spark.rdd.RDD[(String, String)] = ParallelCollectionRDD[114] at parallelize at <console>:29\nres172: Array[(String, (String, String))] = Array((Bacon,(Awesome,Amazing)))\nres174: Array[(String, (String, Option[String]))] = Array((PorkRoll,(Great,None)), (Tofu,(Bogus,None)), (Bacon,(Awesome,Some(Amazing))))\nres175: Array[(String, (Option[String], String))] = Array((Steak,(None,Fine)), (Lettuce,(None,Sad)), (Bacon,(Some(Awesome),Amazing)))\nres176: Array[(String, (Option[String], Option[String]))] = Array((PorkRoll,(Some(Great),None)), (Steak,(None,Some(Fine))), (Tofu,(Some(Bogus),None)), (Lettuce,(None,Some(Sad))), (Bacon,(Some(Awesome),Some(Amazing))))\n"},"dateCreated":"Apr 18, 2016 4:57:37 PM","dateStarted":"Apr 18, 2016 5:02:45 PM","dateFinished":"Apr 18, 2016 5:02:46 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:53"},{"text":"keyValueRDD.groupWith(otherKeyValueRDD).collect()\n","dateUpdated":"Apr 18, 2016 11:11:29 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461013155574_-1102534970","id":"20160418-165915_993773780","result":{"code":"SUCCESS","type":"TEXT","msg":"res180: Array[(String, (Iterable[String], Iterable[String]))] = Array((PorkRoll,(CompactBuffer(Great),CompactBuffer())), (Steak,(CompactBuffer(),CompactBuffer(Fine))), (Tofu,(CompactBuffer(Bogus),CompactBuffer())), (Lettuce,(CompactBuffer(),CompactBuffer(Sad))), (Bacon,(CompactBuffer(Awesome),CompactBuffer(Amazing))))\n"},"dateCreated":"Apr 18, 2016 4:59:15 PM","dateStarted":"Apr 18, 2016 11:11:29 PM","dateFinished":"Apr 18, 2016 11:11:30 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:54"},{"text":"keyValueRDD.cartesian(otherKeyValueRDD).collect()","dateUpdated":"Apr 18, 2016 11:14:49 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461035437842_-1263461216","id":"20160418-231037_1509099531","result":{"code":"SUCCESS","type":"TEXT","msg":"res182: Array[((String, String), (String, String))] = Array(((Bacon,Awesome),(Bacon,Amazing)), ((Bacon,Awesome),(Steak,Fine)), ((Bacon,Awesome),(Lettuce,Sad)), ((PorkRoll,Great),(Bacon,Amazing)), ((PorkRoll,Great),(Steak,Fine)), ((PorkRoll,Great),(Lettuce,Sad)), ((Tofu,Bogus),(Bacon,Amazing)), ((Tofu,Bogus),(Steak,Fine)), ((Tofu,Bogus),(Lettuce,Sad)))\n"},"dateCreated":"Apr 18, 2016 11:10:37 PM","dateStarted":"Apr 18, 2016 11:14:50 PM","dateFinished":"Apr 18, 2016 11:14:50 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:55"},{"text":"%sh cat SMS* | wc -l","dateUpdated":"Apr 18, 2016 11:25:50 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/sh"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461035689890_-367225618","id":"20160418-231449_1615250510","result":{"code":"SUCCESS","type":"TEXT","msg":"    5574\n"},"dateCreated":"Apr 18, 2016 11:14:49 PM","dateStarted":"Apr 18, 2016 11:25:51 PM","dateFinished":"Apr 18, 2016 11:25:51 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:56"},{"text":"keyValueRDD.pipe(\"cut -c2-4\").collect()\n\nkeyValueRDD.coalesce(1).collect()\n\nkeyValueRDD.repartition(2).collect()\n\n\n","dateUpdated":"Apr 19, 2016 9:25:39 AM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461036305040_-398165350","id":"20160418-232505_112344533","result":{"code":"SUCCESS","type":"TEXT","msg":"res237: Array[String] = Array(Bac, Por, Tof)\nres239: Array[(String, String)] = Array((Bacon,Awesome), (PorkRoll,Great), (Tofu,Bogus))\nres241: Array[(String, String)] = Array((Bacon,Awesome), (PorkRoll,Great), (Tofu,Bogus))\n"},"dateCreated":"Apr 18, 2016 11:25:05 PM","dateStarted":"Apr 19, 2016 9:25:39 AM","dateFinished":"Apr 19, 2016 9:25:42 AM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:57","focus":true},{"text":"val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n\nval hereRDD = sc.textFile(\"here\")\nhereRDD.count\nval df = hereRDD.toDF()\ndf.registerTempTable(\"here\")\ndf.printSchema()","dateUpdated":"Apr 19, 2016 12:03:30 AM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461036375489_493205906","id":"20160418-232615_512612527","result":{"code":"SUCCESS","type":"TEXT","msg":"sqlContext: org.apache.spark.sql.SQLContext = org.apache.spark.sql.SQLContext@3746b277\nhereRDD: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[149] at textFile at <console>:30\nres199: Long = 3\ndf: org.apache.spark.sql.DataFrame = [_1: string]\nroot\n |-- _1: string (nullable = true)\n\n"},"dateCreated":"Apr 18, 2016 11:26:15 PM","dateStarted":"Apr 19, 2016 12:03:30 AM","dateFinished":"Apr 19, 2016 12:03:32 AM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:58"},{"text":"%sql\n\nselect * from here","dateUpdated":"Apr 19, 2016 12:04:28 AM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461038416335_827679107","id":"20160419-000016_437447433","result":{"code":"SUCCESS","type":"TABLE","msg":"_1\n(Bacon,Awesome)\n(PorkRoll,Great)\n(Tofu,Bogus)\n","comment":"","msgTable":[[{"value":"(Bacon,Awesome)"}],[{"value":"(PorkRoll,Great)"}],[{"value":"(Tofu,Bogus)"}]],"columnNames":[{"name":"_1","index":0,"aggr":"sum"}],"rows":[["(Bacon,Awesome)"],["(PorkRoll,Great)"],["(Tofu,Bogus)"]]},"dateCreated":"Apr 19, 2016 12:00:16 AM","dateStarted":"Apr 19, 2016 12:04:28 AM","dateFinished":"Apr 19, 2016 12:05:14 AM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:59"},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461038468507_-477327866","id":"20160419-000108_52836889","dateCreated":"Apr 19, 2016 12:01:08 AM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:60","dateUpdated":"Apr 19, 2016 12:45:41 PM","dateFinished":"Apr 19, 2016 12:45:42 PM","dateStarted":"Apr 19, 2016 12:45:41 PM","result":{"code":"SUCCESS","type":"TEXT","msg":"logFile: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[184] at textFile at <console>:29\n"},"text":" val logFile = sc.textFile(\"data/access3.log\")\n ","focus":true},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1461084341753_-61040617","id":"20160419-124541_361907174","dateCreated":"Apr 19, 2016 12:45:41 PM","status":"READY","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:759"}],"name":"Logs","id":"2BK3QYP3A","angularObjects":{"2BFPZ7VPB":[],"2BJ2H4UMU":[],"2BHW3NB2A":[],"2BHU8AK4R":[],"2BG47HRRQ":[],"2BFW2ZBXQ":[],"2BHWJK9CQ":[],"2BGHABJF5":[],"2BKKMDAEP":[],"2BJ7T3PDN":[],"2BG4H7JWC":[],"2BK7Q514A":[],"2BKJAMWS7":[],"2BFYT7ZRD":[]},"config":{"looknfeel":"default"},"info":{}}