Blame view

preprocess.sh 1.53 KB
cfb2c9df1   Joanne   shell
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
  #!/bin/bash
  
  BASEDIR=$(dirname $0)
  
  if [ $# != '1' ];
  then
          echo "usage : sh run.sh <scalaClassPath>"
          echo "ex) sh run.sh com.piki_ds.score.TimeHyperParam"
          exit
  fi
  
  HOME=$BASEDIR
  LOG_DIR=$HOME/logs
  mkdir -p $LOG_DIR
  TODAY=`date +"%Y%m%d"`
  DATE_SUB="30"
  DAY_TO_DELETE="$(date "+%Y%m%d" -d "$DATE_SUB days ago")"
  DELETE_LOG="${LOG_DIR}/$1_$DAY_TO_DELETE.log"
  LOG="${LOG_DIR}/$1_$TODAY.log"
  
  #HADOOP_CONF_DIR=/etc/hadoop/conf
  
  /data/spark/bin/spark-submit \
  --class com.piki_ds.preprocess.$1 \
  --master yarn-client \
be83be97d   Joanne   conf
26
  --executor-memory 3000m \
755e63204   Joanne   preprocess conf c...
27
  --conf "spark.driver.memory=2g" \
be83be97d   Joanne   conf
28
29
30
31
  --conf "spark.dynamicAllocation.minExecutors=90" \
  --conf "spark.dynamicAllocation.maxExecutors=170" \
  --conf "spark.akka.frameSize=2047" \
  --conf "spark.network.timeout=120000" \
755e63204   Joanne   preprocess conf c...
32
  --conf "spark.shuffle.memoryFraction=0.5" \
be83be97d   Joanne   conf
33
  --conf "spark.shuffle.manager=hash" \
cfb2c9df1   Joanne   shell
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
  --conf "spark.default.parallelism=250" \
  $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar  >> $LOG 2>&1
  #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar  >> $LOG 2>&1
  #--jars "lib_managed/jars/mysql/mysql-connector-java/mysql-connector-java-5.1.36.jar" \
  
  
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  
   echo "IF LOG FILE $DELETE_LOG EXISTS DELETE" >> $LOG
  # Log rotation
  if [ -f $DELETE_LOG ]
  then
      rm -f $DELETE_LOG >> $LOG 2>&1
      echo "$DELETE_LOG deleted" >> $LOG
  fi