Commit 140bc42cd19f124b482394b51f0ba59a942bb70b

Authored by Joanne ago
1 parent a7900b4e32
Exists in master

editor score shell options change into new shell scrpt

Showing 2 changed files with 41 additions and 1 deletions Side-by-side Diff

editor_score.sh View file @ 140bc42
  1 +#!/bin/bash
  2 +
  3 +BASEDIR=$(dirname $0)
  4 +
  5 +HOME=$BASEDIR
  6 +LOG_DIR=$HOME/logs
  7 +mkdir -p $LOG_DIR
  8 +TODAY=`date +"%Y%m%d"`
  9 +DATE_SUB="7"
  10 +DAY_TO_DELETE="$(date "+%Y%m%d" -d "$DATE_SUB days ago")"
  11 +DELETE_LOG="${LOG_DIR}/$1_$DAY_TO_DELETE.log"
  12 +LOG="${LOG_DIR}/EditorScore_$TODAY.log"
  13 +
  14 +#HADOOP_CONF_DIR=/etc/hadoop/conf
  15 +
  16 +/data/spark/bin/spark-submit \
  17 +--class com.piki_ds.ver1.$1 \
  18 +--master yarn-client \
  19 +--conf "spark.default.parallelism=250" \
  20 +--conf "spark.driver.memory=2g" \
  21 +--conf "spark.dynamicAllocation.minExecutors=90" \
  22 +--conf "spark.dynamicAllocation.maxExecutors=170" \
  23 +$BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  24 +#target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  25 +#target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  26 +#--jars "lib_managed/jars/mysql/mysql-connector-java/mysql-connector-java-5.1.36.jar" \
  27 +
  28 +echo "END END END END END" >> $LOG
  29 +echo "END END END END END" >> $LOG
  30 +echo "END END END END END" >> $LOG
  31 +echo "END END END END END" >> $LOG
  32 +echo "END END END END END" >> $LOG
  33 +
  34 + echo "IF LOG FILE $DELETE_LOG EXISTS DELETE" >> $LOG
  35 +# Log rotation
  36 +if [ -f $DELETE_LOG ]
  37 +then
  38 + rm -f $DELETE_LOG >> $LOG 2>&1
  39 + echo "$DELETE_LOG deleted" >> $LOG
  40 +fi
ggh_related.sh View file @ 140bc42
... ... @@ -5,6 +5,6 @@
5 5 sh $BASEDIR/mapwithrank.sh
6 6  
7 7 sh $BASEDIR/combinedRun.sh content
8   -sh $BASEDIR/score.sh EditorScore
  8 +sh $BASEDIR/editor_score.sh
9 9 sh $BASEDIR/score.sh QualityScore