Commit 89cc061a0960487f7fd14d6e69f346826a8c7411

Authored by Joanne ago
1 parent be899400e6
Exists in master

insert and exit

Showing 1 changed file with 1 additions and 0 deletions Inline Diff

app/com/piki_ds/ver1/InsertCidList.scala View file @ 89cc061
package com.piki_ds.ver1 1 1 package com.piki_ds.ver1
2 2
import com.piki_ds.utils.hbase.HbaseInserter 3 3 import com.piki_ds.utils.hbase.HbaseInserter
import org.apache.spark.SparkContext 4 4 import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext 5 5 import org.apache.spark.sql.SQLContext
6 6
/** 7 7 /**
* Created by jungwon on 5/12/16. 8 8 * Created by jungwon on 5/12/16.
*/ 9 9 */
10 10
object InsertCidList { 11 11 object InsertCidList {
12 12
var sc: SparkContext = SparkContext.getOrCreate() 13 13 var sc: SparkContext = SparkContext.getOrCreate()
var sqlContext: SQLContext = SQLContext.getOrCreate(sc) 14 14 var sqlContext: SQLContext = SQLContext.getOrCreate(sc)
15 15
//187269,183889,194411 16 16 //187269,183889,194411
17 17
def main(args: Array[String]) { 18 18 def main(args: Array[String]) {
val modelName: Map[String, Seq[String]] = Map( 19 19 val modelName: Map[String, Seq[String]] = Map(
"uniform" -> 20 20 "uniform" ->
Seq("quality"), 21 21 Seq("quality"),
"single"-> 22 22 "single"->
Seq( 23 23 Seq(
/* 24 24 /*
"cf", 25 25 "cf",
"topic", 26 26 "topic",
"age", 27 27 "age",
"sex", 28 28 "sex",
*/ 29 29 */
"w2v" 30 30 "w2v"
), 31 31 ),
"ensemble" -> 32 32 "ensemble" ->
Seq( 33 33 Seq(
"ensemble1", 34 34 "ensemble1",
"ensemble2", 35 35 "ensemble2",
"ensemble3", 36 36 "ensemble3",
"ensemble4" 37 37 "ensemble4"
) 38 38 )
) 39 39 )
modelName("single").foreach(model=> { 40 40 modelName("single").foreach(model=> {
val doi = "20160508" 41 41 val doi = "20160508"
val getMax30 = sc.objectFile[(Int, String)](s"/user/joanne/clols/$doi/$model") 42 42 val getMax30 = sc.objectFile[(Int, String)](s"/user/joanne/clols/$doi/$model")
43 43
getMax30.collect().foreach(x=>{ 44 44 getMax30.collect().foreach(x=>{
val tableName = s"uuid-cidlist_$model" 45 45 val tableName = s"uuid-cidlist_$model"
val insertArray = Array((x._1.toString, x._2)) 46 46 val insertArray = Array((x._1.toString, x._2))
val test = new HbaseInserter(tableName) 47 47 val test = new HbaseInserter(tableName)