Commit d89affd590fc53fd33d86e30c2cbb5ea0dcd1b0b

Authored by aram ago
1 parent dca1584aec
Exists in master

map with rank conf add

Showing 1 changed file with 2 additions and 0 deletions Inline Diff

mapwithrank.sh View file @ d89affd
#!/bin/bash 1 1 #!/bin/bash
2 2
BASEDIR=$(dirname $0) 3 3 BASEDIR=$(dirname $0)
4 4
HOME=$BASEDIR 5 5 HOME=$BASEDIR
LOG_DIR=$HOME/logs 6 6 LOG_DIR=$HOME/logs
mkdir -p $LOG_DIR 7 7 mkdir -p $LOG_DIR
TODAY=`date +"%Y%m%d"` 8 8 TODAY=`date +"%Y%m%d"`
DATE_SUB="10" 9 9 DATE_SUB="10"
DAY_TO_DELETE="$(date "+%Y%m%d" -d "$DATE_SUB days ago")" 10 10 DAY_TO_DELETE="$(date "+%Y%m%d" -d "$DATE_SUB days ago")"
DELETE_LOG="${LOG_DIR}/MapWithRank_$DAY_TO_DELETE.log" 11 11 DELETE_LOG="${LOG_DIR}/MapWithRank_$DAY_TO_DELETE.log"
LOG="${LOG_DIR}/MapWithRank_$TODAY.log" 12 12 LOG="${LOG_DIR}/MapWithRank_$TODAY.log"
13 13
#HADOOP_CONF_DIR=/etc/hadoop/conf 14 14 #HADOOP_CONF_DIR=/etc/hadoop/conf
15 15
/data/spark/bin/spark-submit \ 16 16 /data/spark/bin/spark-submit \
--class com.piki_ds.preprocess.MapWithRank \ 17 17 --class com.piki_ds.preprocess.MapWithRank \
--master yarn-client \ 18 18 --master yarn-client \
19 --executor-memory 4000m \
20 --driver-memory 5000m \
$BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 19 21 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
#target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 20 22 #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
#target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 21 23 #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
#--jars "lib_managed/jars/mysql/mysql-connector-java/mysql-connector-java-5.1.36.jar" \ 22 24 #--jars "lib_managed/jars/mysql/mysql-connector-java/mysql-connector-java-5.1.36.jar" \
23 25
24 26
echo "END END END END END" >> $LOG 25 27 echo "END END END END END" >> $LOG
echo "END END END END END" >> $LOG 26 28 echo "END END END END END" >> $LOG
echo "END END END END END" >> $LOG 27 29 echo "END END END END END" >> $LOG
echo "END END END END END" >> $LOG 28 30 echo "END END END END END" >> $LOG
echo "END END END END END" >> $LOG 29 31 echo "END END END END END" >> $LOG
30 32
echo "IF LOG FILE $DELETE_LOG EXISTS DELETE" >> $LOG 31 33 echo "IF LOG FILE $DELETE_LOG EXISTS DELETE" >> $LOG