I am using sparklauncher to start a spark application. In spark application I am inserting data into hive table and using some hbase-hive linked table in join query. I have added hive-hbase-handler-1.1.0-cdh5.13.0.jar in the spark launcher, still I ma getting Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat i have added jar in sparklauncher error:
def launch(hdfsFilePath:String):Unit={
println("Inside ApplicationLauncher")
val command = new SparkLauncher()
.setAppResource("/home/cloudera/Desktop/Avi/LiveProjects/MusicDataAnalysis/target/scala-2.11/musicdataanalysis_2.11-0.1.jar")
.setMainClass("ParseInputFile")
.setVerbose(false)
.addAppArgs(hdfsFilePath)
.setMaster("local")
.addJar("/home/cloudera/Desktop/Avi/jars/hive-hbase-handler-1.1.0-cdh5.13.0.jar")
.addJar("/home/cloudera/Desktop/Avi/jars/spark-xml_2.11-0.5.0.jar")
println("Done with Spark Launcher")
val appHandle = command.startApplication()
appHandle.addListener(new SparkAppHandle.Listener{
def infoChanged(sparkAppHandle : SparkAppHandle) : Unit = {
// println(sparkAppHandle.getState + " Custom Print")
}
def stateChanged(sparkAppHandle : SparkAppHandle) : Unit = {
println(sparkAppHandle.getState)
if ("FINISHED".equals(sparkAppHandle.getState.toString)){
sparkAppHandle.stop
}
}
})
My problem got solved when I have added hbase-0.92.1.jar along with hive-hbase-handler-1.1.0-cdh5.13.0.jar. Please find the below working code:
def launch(hdfsFilePath:String):Unit={
println("Inside ApplicationLauncher")
val command = new SparkLauncher()
.setAppResource("/home/cloudera/Desktop/Avi/LiveProjects/MusicDataAnalysis/target/scala-2.11/musicdataanalysis_2.11-0.1.jar")
.setMainClass("ParseInputFile")
.setVerbose(false)
.addAppArgs(hdfsFilePath)
.setMaster("local")
.addJar("file:///home/cloudera/Desktop/Avi/jars/hbase-0.92.1.jar")
.addJar("file:///home/cloudera/Desktop/Avi/jars/hive-hbase-handler-3.1.1.jar")
.addJar("file:///home/cloudera/Desktop/Avi/jars/spark-xml_2.11-0.5.0.jar")
println("Done with Spark Launcher")
val appHandle = command.startApplication()
appHandle.addListener(new SparkAppHandle.Listener{
def infoChanged(sparkAppHandle : SparkAppHandle) : Unit = {
// println(sparkAppHandle.getState + " Custom Print")
}
def stateChanged(sparkAppHandle : SparkAppHandle) : Unit = {
println(sparkAppHandle.getState)
if ("FINISHED".equals(sparkAppHandle.getState.toString)){
sparkAppHandle.stop
}
}
})
}