Commit ad21ba55657d0bd80aebe69acb4635c5e67b0614

Authored by Joanne ago
1 parent 35ad293621
Exists in master

partition and sleep

Showing 1 changed file with 2 additions and 1 deletions Side-by-side Diff

app/com/piki_ds/ver1/InsertCidList.scala View file @ ad21ba5
... ... @@ -40,11 +40,12 @@
40 40 modelName("single").foreach(model=> {
41 41 val doi = "20160508"
42 42 val getMax30 = sc.objectFile[(Int, String)](s"/user/joanne/clols/$doi/$model")
43   - getMax30.groupBy(x=>x._1%1000).foreach(x=>{
  43 + getMax30.groupBy(x=>x._1%5000).foreach(x=>{
44 44 val tableName = s"uuid-cidlist_$model"
45 45 val insertArray = x._2.map(a=>(a._1.toString,a._2)).toArray
46 46 val test = new HbaseInserter(tableName)
47 47 test.insert(insertArray)
  48 + Thread.sleep(3000)
48 49 })
49 50 })
50 51 }