Blame view

editor_score.sh 1.21 KB
140bc42cd   Joanne   editor score shel...
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
  #!/bin/bash
  
  BASEDIR=$(dirname $0)
  
  HOME=$BASEDIR
  LOG_DIR=$HOME/logs
  mkdir -p $LOG_DIR
  TODAY=`date +"%Y%m%d"`
  DATE_SUB="7"
  DAY_TO_DELETE="$(date "+%Y%m%d" -d "$DATE_SUB days ago")"
  DELETE_LOG="${LOG_DIR}/$1_$DAY_TO_DELETE.log"
  LOG="${LOG_DIR}/EditorScore_$TODAY.log"
  
  #HADOOP_CONF_DIR=/etc/hadoop/conf
  
  /data/spark/bin/spark-submit \
0a5c9fb1f   Joanne   editor score
17
  --class com.piki_ds.ver1.EditorScore \
140bc42cd   Joanne   editor score shel...
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
  --master yarn-client \
  --conf "spark.default.parallelism=250" \
  --conf "spark.driver.memory=2g" \
  --conf "spark.dynamicAllocation.minExecutors=90" \
  --conf "spark.dynamicAllocation.maxExecutors=170" \
  $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar  >> $LOG 2>&1
  #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar  >> $LOG 2>&1
  #--jars "lib_managed/jars/mysql/mysql-connector-java/mysql-connector-java-5.1.36.jar" \
  
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  
   echo "IF LOG FILE $DELETE_LOG EXISTS DELETE" >> $LOG
  # Log rotation
  if [ -f $DELETE_LOG ]
  then
      rm -f $DELETE_LOG >> $LOG 2>&1
      echo "$DELETE_LOG deleted" >> $LOG
  fi