Skip to content

Commit 13f5e8b

Browse files
holdenkholdenk
committed
Revert "Revert "upgrade to 1.3""
This reverts commit b824b15.
1 parent a32c9a8 commit 13f5e8b

5 files changed

Lines changed: 15 additions & 14 deletions

File tree

‎build-project‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,11 @@ cd mini-complete-example
88
echo $PWD && mvn clean && mvn compile
99
cd ..
1010
# Run the tests
11-
export SPARK_HOME=./spark-1.1.0-bin-hadoop2.4/
11+
export SPARK_HOME=./spark-1.3.0-bin-hadoop1/
1212
./sbt/sbt compile package assembly
1313
echo $?
1414
time ./run-all-examples
1515
echo $?
1616
echo "done"
1717
# Try and build with maven, skip for now
18-
#mvn clean && mvn compile && mvn package
18+
#mvn clean && mvn compile && mvn package

‎build.sbt‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,13 @@ seq(sbtprotobuf.ProtobufPlugin.protobufSettings: _*)
1515

1616
// additional libraries
1717
libraryDependencies ++= Seq(
18-
"org.apache.spark" %% "spark-core" % "1.1.1" % "provided",
19-
"org.apache.spark" %% "spark-sql" % "1.1.1" % "provided",
20-
"org.apache.spark" %% "spark-hive" % "1.1.1" % "provided",
21-
"org.apache.spark" %% "spark-streaming" % "1.1.1",
22-
"org.apache.spark" %% "spark-streaming-kafka" % "1.1.1",
23-
"org.apache.spark" %% "spark-streaming-flume" % "1.1.1",
24-
"org.apache.spark" %% "spark-mllib" % "1.1.1",
18+
"org.apache.spark" %% "spark-core" % "1.3.0" % "provided",
19+
"org.apache.spark" %% "spark-sql" % "1.3.0",
20+
"org.apache.spark" %% "spark-hive" % "1.3.0",
21+
"org.apache.spark" %% "spark-streaming" % "1.3.0",
22+
"org.apache.spark" %% "spark-streaming-kafka" % "1.3.0",
23+
"org.apache.spark" %% "spark-streaming-flume" % "1.3.0",
24+
"org.apache.spark" %% "spark-mllib" % "1.3.0",
2525
"org.apache.commons" % "commons-lang3" % "3.0",
2626
"org.eclipse.jetty" % "jetty-client" % "8.1.14.v20131031",
2727
"com.typesafe.play" % "play-json_2.10" % "2.2.1",

‎setup-project‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ echo "deb http://debian.datastax.com/community stable main" | sudo tee -a /etc/a
99
curl -L http://debian.datastax.com/debian/repo_key | sudo apt-key add -
1010
sudo apt-get update > aptlog &
1111
APT_GET_UPDATE_PID=$!
12-
axel http://d3kbcqa49mib13.cloudfront.net/spark-1.1.0-bin-hadoop2.4.tgz > sparkdl &
12+
axel http://d3kbcqa49mib13.cloudfront.net/spark-1.3.0-bin-hadoop1.tgz > sparkdl &
1313
SPARK_DL_PID=$!
1414
axel http://mirrors.ibiblio.org/apache/kafka/0.8.1.1/kafka_2.9.2-0.8.1.1.tgz > kafkadl &
1515
KAFKA_DL_PID=$!
@@ -20,7 +20,7 @@ sudo mkdir -p /etc/apt/sources.list.d/
2020
echo "install urllib3"
2121
sudo pip install urllib3
2222
wait $SPARK_DL_PID || echo "Spark DL finished early"
23-
tar -xf spark-1.1.0-bin-hadoop2.4.tgz
23+
tar -xf spark-1.3.0-bin-hadoop1.tgz
2424
wait $APT_GET_UPDATE_PID
2525
echo "Installing protobuf"
2626
sudo apt-get install protobuf-compiler

‎src/main/scala/com/oreilly/learningsparkexamples/scala/SparkSQLTwitter.scala‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import org.apache.spark._
77
import org.apache.spark.SparkContext._
88
import org.apache.spark.sql.hive.HiveContext
99

10+
1011
case class HappyPerson(handle: String, favouriteBeverage: String)
1112

1213
object SparkSQLTwitter {
@@ -26,7 +27,7 @@ object SparkSQLTwitter {
2627
conf.set("spark.sql.inMemoryColumnarStorage.batchSize", batchSize)
2728
val sc = new SparkContext(conf)
2829
val hiveCtx = new HiveContext(sc)
29-
import hiveCtx._
30+
import hiveCtx.implicits._
3031
// Load some tweets
3132
val input = hiveCtx.jsonFile(inputFile)
3233
// Print the schema
@@ -42,7 +43,7 @@ object SparkSQLTwitter {
4243
val happyPeopleRDD = sc.parallelize(List(HappyPerson("holden", "coffee")))
4344
happyPeopleRDD.registerTempTable("happy_people")
4445
// UDF
45-
registerFunction("strLenScala", (_: String).length)
46+
hiveCtx.udf().register("strLenScala", (_: String).length)
4647
val tweetLength = hiveCtx.sql("SELECT strLenScala('tweet') FROM tweets LIMIT 10")
4748
tweetLength.collect().map(println(_))
4849
// Two sums at once (crazy town!)

‎src/python/SparkSQLTwitter.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
happyPeopleSchemaRDD = hiveCtx.inferSchema(happyPeopleRDD)
2222
happyPeopleSchemaRDD.registerTempTable("happy_people")
2323
# Make a UDF to tell us how long some text is
24-
hiveCtx.registerFunction("strLenPython", lambda x: len(x), IntegerType())
24+
hiveCtx.udf().register("strLenPython", lambda x: len(x), IntegerType())
2525
lengthSchemaRDD = hiveCtx.sql("SELECT strLenPython('text') FROM tweets LIMIT 10")
2626
print lengthSchemaRDD.collect()
2727
sc.stop()

0 commit comments

Comments
 (0)