Commit 695753ed8b75abaac00ebd3bb301a8e8ff56155b

Authored by joel ago
1 parent 50c3645b29
Exists in master

initial setting

Showing 2 changed files with 16 additions and 36 deletions Side-by-side Diff

app/com/piki_ds/utils/hbase/PikiHbase.scala View file @ 695753e
... ... @@ -105,12 +105,17 @@
105 105 try {
106 106 val admin =new HBaseAdmin(PikiHbaseConfig.conf)
107 107 val tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName))
108   - admin.disableTable(tableName)
  108 +
  109 + if(admin.isTableEnabled(tableName)) admin.disableTable(tableName)
  110 +
109 111 cfNames.map(cf => {
110 112 val family = new HColumnDescriptor(cf)
111 113 family.setCompressionType(Algorithm.SNAPPY)
112 114 tableDescriptor.addFamily(family)
113 115 })
  116 +
  117 + if(admin.isTableDisabled(tableName)) admin.enableTable(tableName)
  118 +
114 119 } catch {
115 120 case e: Exception => {
116 121 log_err.error(s"addColumnFamily error hbase table: $tableName")
117 122  
118 123  
... ... @@ -121,10 +126,17 @@
121 126  
122 127 def createTable(tableName: String): Unit = {
123 128 try {
  129 + val default_Column_Family_Name = "f1"
124 130 val admin = new HBaseAdmin(PikiHbaseConfig.conf)
125 131 val tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName))
  132 + val defaultFamily = new HColumnDescriptor(default_Column_Family_Name)
  133 + //defaultFamily.setCompressionType(Algorithm.LZO)
  134 +
  135 + tableDescriptor.addFamily(defaultFamily)
126 136 admin.createTable(tableDescriptor)
127   - admin.enableTable(tableName)
  137 +
  138 + if(admin.isTableDisabled(tableName)) admin.enableTable(tableName)
  139 +
128 140 logger.info(s"table created $tableName")
129 141 } catch {
130 142 case e: Exception => {
131 143  
... ... @@ -239,10 +251,9 @@
239 251  
240 252 table.close()
241 253  
242   - Thread.sleep(3000)
243   -
244 254 //create table readHistory
245   - //createTable("readHistory")
  255 + //dropTable("readHistory")
  256 + createTable("readHistory")
246 257 //addColumnFamily("readHistory", List("f1"))
247 258  
248 259 PikiHbase.listTables(connection).foreach(println)
... ... @@ -15,25 +15,17 @@
15 15  
16 16 val sprayV = "1.3.3"
17 17  
18   -val sparkV = "1.3.1"
19   -
20 18 //Play2WarKeys.servletVersion := "3.0"
21 19  
22   -
23 20 //Play2WarPlugin.play2WarSettings
24 21  
25 22 //net.virtualvoid.sbt.graph.Plugin.graphSettings
26 23  
27   -val hadoopversion = "2.6.0"
28   -
29 24 libraryDependencies ++= Seq(
30 25 jdbc,
31 26 anorm,
32 27 cache,
33 28 ws,
34   -"org.apache.hadoop" % "hadoop-hdfs" % hadoopversion exclude("commons-daemon", "commons-daemon"),
35   - "org.apache.hadoop" % "hadoop-auth" % hadoopversion exclude("commons-daemon", "commons-daemon"),
36   - "org.apache.hadoop" % "hadoop-client" % hadoopversion exclude("commons-daemon", "commons-daemon"),
37 29 "org.slf4j" % "slf4j-api" % "1.7.2",
38 30 "com.typesafe.akka" %% "akka-actor" % "2.3.9",
39 31 "com.typesafe.akka" %% "akka-remote" % "2.3.9",
... ... @@ -42,28 +34,6 @@
42 34 "io.spray" %% "spray-caching" % sprayV,
43 35 "io.spray" %% "spray-routing" % sprayV,
44 36 "io.spray" %% "spray-client" % sprayV,
45   - "com.google.code.findbugs" % "jsr305" % "2.0.3",
46   - "com.mchange" % "c3p0" % "0.9.5",
47   - "com.google.guava" % "guava" % "18.0",
48   - "mysql" % "mysql-connector-java" % "5.1.32",
49   - "org.json4s" %% "json4s-jackson" % "3.2.11",
50   - "org.json4s" %% "json4s-ext" % "3.2.11",
51   - "org.json4s" %% "json4s-native" % "3.2.11",
52   - "org.mariadb.jdbc" % "mariadb-java-client" % "1.1.8",
53   - "com.netaporter" %% "scala-uri" % "0.4.7",
54   - "org.feijoas" %% "mango" % "0.11",
55   - "com.twitter.penguin" % "korean-text" % "4.1.3piki-SNAPSHOT",
56   - "com.piki_ds" %% "dsactors" % "0.1.0-SNAPSHOT",
57   - "com.piki_ds" %% "dspikirepository" % "0.1.0-SNAPSHOT",
58   - "com.twitter" %% "util-collection" % "6.23.0",
59   - "org.scalatest" %% "scalatest" % "2.1.6" % "test",
60   - "org.apache.spark" %% "spark-core" % sparkV,
61   - "org.apache.spark" %% "spark-streaming" % sparkV,
62   - "org.apache.spark" %% "spark-sql" % sparkV,
63   - "org.apache.spark" %% "spark-mllib" % sparkV,
64   - "org.apache.spark" %% "spark-repl" % sparkV,
65   - "org.apache.spark" %% "spark-yarn" % sparkV,
66   - "com.typesafe.akka" %% "akka-testkit" % "2.3.4" % "test",
67 37 "org.apache.hbase" % "hbase-client" % "0.98.6-hadoop2",
68 38 "org.apache.hbase" % "hbase-common" % "0.98.6-hadoop2"
69 39 )
... ... @@ -73,7 +43,6 @@
73 43 "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/",
74 44 "Sonatype Releases" at "https://oss.sonatype.org/content/repositories/releases/",
75 45 "Scala Tools Snapshots" at "http://scala-tools.org/repo-snapshots/",
76   - "twitter" at "http://maven.twttr.com/",
77 46 "spray repo" at "http://repo.spray.io/",
78 47 "Apache HBase" at "https://repository.apache.org/content/repositories/releases"
79 48 )