Commit fc76c64827541e7df7094b7d43983d39ea5fa14c

Authored by joel ago
1 parent 822af0d1dc
Exists in master

initial setting

Showing 3 changed files with 16 additions and 60 deletions Side-by-side Diff

app/com/piki_ds/utils/hbase/PikiHbase.scala View file @ fc76c64
1 1 package com.piki_ds.utils.hbase
2 2  
3   -import com.piki.ds.utils.Base62
4 3 import org.apache.hadoop.hbase.{HColumnDescriptor, TableName, HTableDescriptor}
5 4 import org.apache.hadoop.hbase.client._
6 5 import org.apache.hadoop.hbase.filter.{PrefixFilter}
... ... @@ -126,6 +125,7 @@
126 125 }
127 126 }
128 127  
  128 + //disable -> drop -> create -> enable
129 129 def truncateTable(tableName: String): Unit = {
130 130 try {
131 131 val admin = new HBaseAdmin(PikiHbaseConfig.conf)
... ... @@ -178,58 +178,15 @@
178 178  
179 179 //(1L to 99999999).map({ x =>
180 180 (1L to 2).map({ x =>
181   - val l = s""""SESSION|START|1430677627828|39045405|A2.1.3|samsung-ks01lteskt|Android 4.4.2|2
182   -COMMON|LOAD|1430677630274|39045405|m|{"contents":[{"h":[12399]},{"m":[96732,96114,96132,95832,96516,97710,97632,95400,97188,96378,97644,97662,93144,96612]},{"r1":[21,22,23,44,58,68,103,104,119,125,126,128,133,158,167,169,170,171,172,182,183,184,186,192,197,213,215,521,532,536,540,544,546,552,554,563,567,572,602,605,609,617,618,619,635,651,652,685,697,703,1040,1602,2093,2210,2744,3079,3602,3749,5122,5407,5735,9375,11849,12219,12477,12909,16495,16832,17945,20939,21067,22601,22835,26310,31320,32399,32413,44058,46659,55307,55437,55958,57554,60969,62100,63512,64178,72036,75468,75906,76878,78336,78936,86225,87077,89171,89819,91175,92213,94350]},{"m":[97230,96360,96456,97032,97614,96762]}]}
183   -CONTENT|OPEN|1438306705052|43189353|126805|m|1438306694460|8|TOON
184   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
185   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
186   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
187   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
188   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
189   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
190   -CONTENT|EXPOSURE|1430677630477|39045405|96132|3583|121|m
191   -CONTENT|EXPOSURE|1430677630440|39045405|96114|3620|127|m
192   -CONTENT|EXPOSURE|1430677630407|39045405|96732|3654|143|m
193   -CONTENT|EXPOSURE|1430677630532|39045405|95832|3529|114|m
194   -CONTENT|EXPOSURE|1430677630304|39045405|12399|3757|85|h
195   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
196   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
197   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
198   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
199   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
200   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
201   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
202   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
203   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
204   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
205   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
206   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
207   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
208   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
209   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
210   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
211   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
212   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
213   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
214   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
215   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
216   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
217   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
218   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
219   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
220   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
221   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
222   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
223   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
224   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
225   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
226   -CONTENT|OPEN|1437700722042|${math.abs(Random.nextLong())}|${math.abs(Random.nextLong())}|m
227   -CARD|CONSUME|1430677732367|39045405|96132|14|2935
228   -CARD|CONSUME|1430677738495|39045405|96132|15|6127
229   -CARD|CONSUME|1430677743225|39045405|96132|16|4724
230   -CARD|CONSUME|1430677745326|39045405|96132|17|2099
231   -CONTENT|EXPOSURE|1430677752780|39045405|97632|2593|86|m
232   -CONTENT|EXPOSURE|1430677751880|39045405|97710|3493|34|m"""
  181 + val l = s"""CONTENT|OPEN|1438304264373|10896834|126075|m|1438304261545|1|ALBUM
  182 +CONTENT|OPEN|1438304325268|10896834|125915|m|1438304261545|4|ALBUM
  183 +CONTENT|OPEN|1438304316412|32367650|123370|m|1438304312151|3|ALBUM
  184 +CONTENT|OPEN|1438304242489|239594|126075|m|1438304233109|1|ALBUM
  185 +CONTENT|OPEN|1438304286992|239594|123370|m|1438304233109|3|ALBUM
  186 +CONTENT|OPEN|1438304321491|14312740|126075|m|1438304314261|1|ALBUM
  187 +CONTENT|OPEN|1438302147525|31262663|122000|m|1438301840341|29|CHST
  188 +CONTENT|OPEN|1438302172883|31262663|126095|m|1438302165410|39|ALBUM
  189 +CONTENT|OPEN|1438302329364|31262663|124040|m|1438302165410|43|ALBUM"""
233 190  
234 191 val lines = l.split("\n").map(x => x.trim)
235 192 println(1)
236 193  
... ... @@ -238,13 +195,9 @@
238 195  
239 196 })
240 197  
241   - println(Base62.encode(6030146367697000226L))
242   - println(Base62.encode(6022397113819400614L))
243   - println(Base62.encode(6929227236222314801L))
244 198 println("put All end")
245 199  
246 200 val uuid = "6055485486985007966"
247   - println(Base62.decode("7DKBdnuNl78"))
248 201 //val result2 = PikiHbase.scanWithFilter(connection, table, uuid)
249 202  
250 203 //val value2 = result2.value()
app/com/piki_ds/utils/hbase/PikiHbaseConfig.scala View file @ fc76c64
... ... @@ -36,5 +36,5 @@
36 36 outResource.close()
37 37 tmpFile
38 38 }
39   -}`
  39 +}
... ... @@ -63,7 +63,9 @@
63 63 "org.apache.spark" %% "spark-mllib" % sparkV,
64 64 "org.apache.spark" %% "spark-repl" % sparkV,
65 65 "org.apache.spark" %% "spark-yarn" % sparkV,
66   - "com.typesafe.akka" %% "akka-testkit" % "2.3.4" % "test"
  66 + "com.typesafe.akka" %% "akka-testkit" % "2.3.4" % "test",
  67 + "org.apache.hbase" % "hbase-client" % "0.98.6-hadoop2",
  68 + "org.apache.hbase" % "hbase-common" % "0.98.6-hadoop2"
67 69 )
68 70  
69 71  
... ... @@ -72,7 +74,8 @@
72 74 "Sonatype Releases" at "https://oss.sonatype.org/content/repositories/releases/",
73 75 "Scala Tools Snapshots" at "http://scala-tools.org/repo-snapshots/",
74 76 "twitter" at "http://maven.twttr.com/",
75   - "spray repo" at "http://repo.spray.io/"
  77 + "spray repo" at "http://repo.spray.io/",
  78 + "Apache HBase" at "https://repository.apache.org/content/repositories/releases"
76 79 )
77 80  
78 81 resolvers += "Local Maven Repository" at "file://"+Path.userHome.absolutePath+"/.m2/repository"