Commit e53cc1e5e1a26560d9599d1f1845f6c4aac9211e

Authored by Joanne ago
1 parent ad21ba5565
Exists in master

coalesce

Showing 1 changed file with 1 additions and 1 deletions Side-by-side Diff

app/com/piki_ds/ver1/InsertCidList.scala View file @ e53cc1e
... ... @@ -40,7 +40,7 @@
40 40 modelName("single").foreach(model=> {
41 41 val doi = "20160508"
42 42 val getMax30 = sc.objectFile[(Int, String)](s"/user/joanne/clols/$doi/$model")
43   - getMax30.groupBy(x=>x._1%5000).foreach(x=>{
  43 + getMax30.coalesce(5).groupBy(x=>x._1%5000).foreach(x=>{
44 44 val tableName = s"uuid-cidlist_$model"
45 45 val insertArray = x._2.map(a=>(a._1.toString,a._2)).toArray
46 46 val test = new HbaseInserter(tableName)