data science team / dsquality

Blame view

gghVer2.sh 1.48 KB
edit raw normal view history
d8c6c1a5d   evan   arg change and ba... 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
 
  #!/usr/bin/env
  
  BASEDIR=$(dirname $0)
  
  if [ $# != '1' ] && [ $# != '2' ];
  then
          echo "usage : sh score.sh <scalaClassName>"
          echo "ex) sh score.sh EditorScore"
          exit
  fi
  
  HOME=$BASEDIR
  LOG_DIR=$HOME/logs
  mkdir -p $LOG_DIR
  TODAY=`date +"%Y%m%d"`
  DATE_SUB="7"
  DAY_TO_DELETE="$(date "+%Y%m%d" -d "$DATE_SUB days ago")"
  DELETE_LOG="${LOG_DIR}/$1_$DAY_TO_DELETE.log"
  LOG="${LOG_DIR}/$1_$2_$TODAY.log"
  
  #HADOOP_CONF_DIR=/etc/hadoop/conf
  
  /data/spark/bin/spark-submit \
  --class $1 \
  --master yarn-client \
3a990d5ee   evan   sh memory 3000 an... 
26
27
28
29
30
31
32
 
  --executor-memory 3000m \
  --conf "spark.dynamicAllocation.minExecutors=90" \
  --conf "spark.dynamicAllocation.maxExecutors=170" \
  --conf "spark.akka.frameSize=2047" \
  --conf "spark.network.timeout=120000" \
  --conf "spark.shuffle.memoryFraction=0.5" \
  --conf "spark.shuffle.manager=hash" \
d8c6c1a5d   evan   arg change and ba... 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
  --conf "spark.default.parallelism=250" \
  $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar $2 >> $LOG 2>&1
  #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar  >> $LOG 2>&1
  #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar  >> $LOG 2>&1
  #--jars "lib_managed/jars/mysql/mysql-connector-java/mysql-connector-java-5.1.36.jar" \
  
  
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  
   echo "IF LOG FILE $DELETE_LOG EXISTS DELETE" >> $LOG
  # Log rotation
  if [ -f $DELETE_LOG ]
  then
      rm -f $DELETE_LOG >> $LOG 2>&1
      echo "$DELETE_LOG deleted" >> $LOG
  fi