Commit 0a5c9fb1f64c9fe3218ce8fd773a163c456a0775

Authored by Joanne ago
1 parent 2917ca52b2
Exists in master

editor score

Showing 1 changed file with 1 additions and 1 deletions Inline Diff

editor_score.sh View file @ 0a5c9fb
#!/bin/bash 1 1 #!/bin/bash
2 2
BASEDIR=$(dirname $0) 3 3 BASEDIR=$(dirname $0)
4 4
HOME=$BASEDIR 5 5 HOME=$BASEDIR
LOG_DIR=$HOME/logs 6 6 LOG_DIR=$HOME/logs
mkdir -p $LOG_DIR 7 7 mkdir -p $LOG_DIR
TODAY=`date +"%Y%m%d"` 8 8 TODAY=`date +"%Y%m%d"`
DATE_SUB="7" 9 9 DATE_SUB="7"
DAY_TO_DELETE="$(date "+%Y%m%d" -d "$DATE_SUB days ago")" 10 10 DAY_TO_DELETE="$(date "+%Y%m%d" -d "$DATE_SUB days ago")"
DELETE_LOG="${LOG_DIR}/$1_$DAY_TO_DELETE.log" 11 11 DELETE_LOG="${LOG_DIR}/$1_$DAY_TO_DELETE.log"
LOG="${LOG_DIR}/EditorScore_$TODAY.log" 12 12 LOG="${LOG_DIR}/EditorScore_$TODAY.log"
13 13
#HADOOP_CONF_DIR=/etc/hadoop/conf 14 14 #HADOOP_CONF_DIR=/etc/hadoop/conf
15 15
/data/spark/bin/spark-submit \ 16 16 /data/spark/bin/spark-submit \
--class com.piki_ds.ver1.EditorSore \ 17 17 --class com.piki_ds.ver1.EditorScore \
--master yarn-client \ 18 18 --master yarn-client \
--conf "spark.default.parallelism=250" \ 19 19 --conf "spark.default.parallelism=250" \
--conf "spark.driver.memory=2g" \ 20 20 --conf "spark.driver.memory=2g" \
--conf "spark.dynamicAllocation.minExecutors=90" \ 21 21 --conf "spark.dynamicAllocation.minExecutors=90" \
--conf "spark.dynamicAllocation.maxExecutors=170" \ 22 22 --conf "spark.dynamicAllocation.maxExecutors=170" \
$BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 23 23 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
#target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 24 24 #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
#target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 25 25 #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
#--jars "lib_managed/jars/mysql/mysql-connector-java/mysql-connector-java-5.1.36.jar" \ 26 26 #--jars "lib_managed/jars/mysql/mysql-connector-java/mysql-connector-java-5.1.36.jar" \
27 27
echo "END END END END END" >> $LOG 28 28 echo "END END END END END" >> $LOG
echo "END END END END END" >> $LOG 29 29 echo "END END END END END" >> $LOG
echo "END END END END END" >> $LOG 30 30 echo "END END END END END" >> $LOG
echo "END END END END END" >> $LOG 31 31 echo "END END END END END" >> $LOG
echo "END END END END END" >> $LOG 32 32 echo "END END END END END" >> $LOG
33 33
echo "IF LOG FILE $DELETE_LOG EXISTS DELETE" >> $LOG 34 34 echo "IF LOG FILE $DELETE_LOG EXISTS DELETE" >> $LOG
# Log rotation 35 35 # Log rotation