Commit 0ae4b22e9439142435746b09ca08f1583af8f2b5

Authored by Joanne ago
1 parent 40ac8fc5b6
Exists in master

collect

Showing 1 changed file with 3 additions and 2 deletions Side-by-side Diff

app/com/piki_ds/ver1/InsertCidList.scala View file @ 0ae4b22
... ... @@ -40,9 +40,10 @@
40 40 modelName("single").foreach(model=> {
41 41 val doi = "20160508"
42 42 val getMax30 = sc.objectFile[(Int, String)](s"/user/joanne/clols/$doi/$model")
43   - getMax30.coalesce(5).groupBy(x=>x._1%5000).foreach(x=>{
  43 +
  44 + getMax30.collect().groupBy(x=>x._1%5000).foreach(x=>{
44 45 val tableName = s"uuid-cidlist_$model"
45   - val insertArray = x._2.map(a=>(a._1.toString,a._2)).toArray
  46 + val insertArray = x._2.map(a=>(a._1.toString,a._2))
46 47 val test = new HbaseInserter(tableName)
47 48 test.insert(insertArray)
48 49 Thread.sleep(1000+x._1)