Commit cfb2c9df16938dcc5629439b5719dad38cffa0fb

Authored by Joanne ago
1 parent 8799894620
Exists in master

shell

Showing 3 changed files with 56 additions and 11 deletions Side-by-side Diff

combinedRun.sh View file @ cfb2c9d
... ... @@ -4,16 +4,16 @@
4 4  
5 5 if [ $1 == 'time' ]
6 6 then
7   - sh $BASEDIR/score.sh com.piki_ds.ver1.TimeScore
8   - sh $BASEDIR/score.sh com.piki_ds.ver1.TimeHyperParam
  7 + sh $BASEDIR/score.sh TimeScore
  8 + sh $BASEDIR/score.sh TimeHyperParam
9 9 fi
10 10  
11 11 if [ $1 == 'content' ]
12 12 then
13   - sh $BASEDIR/score.sh com.piki_ds.ver1.DashRelatedScore
  13 + sh $BASEDIR/score.sh DashRelatedScore
14 14  
15   - sh $BASEDIR/score.sh com.piki_ds.ver1.ConRejectScore
16   - sh $BASEDIR/score.sh com.piki_ds.ver1.ConEfficiencyScore
17   - sh $BASEDIR/score.sh com.piki_ds.ver1.ContentScore
  15 + sh $BASEDIR/score.sh ConRejectScore
  16 + sh $BASEDIR/score.sh ConEfficiencyScore
  17 + sh $BASEDIR/score.sh ContentScore
18 18 fi
ggh_related.sh View file @ cfb2c9d
1 1 #!/bin/bash
2 2  
3 3 BASEDIR=$(dirname $0)
4   -sh $BASEDIR/score.sh com.piki_ds.preprocess.MakingLapse
5   -sh $BASEDIR/score.sh com.piki_ds.preprocess.MapWithRank
  4 +sh $BASEDIR/preprocess.sh MakingLapse
  5 +sh $BASEDIR/preprocess.sh MapWithRank
6 6  
7 7 sh $BASEDIR/combinedRun.sh content
8   -sh $BASEDIR/score.sh com.piki_ds.ver1.EditorScore
9   -sh $BASEDIR/score.sh com.piki_ds.ver1.QualityScore
  8 +sh $BASEDIR/score.sh EditorScore
  9 +sh $BASEDIR/score.sh QualityScore
preprocess.sh View file @ cfb2c9d
  1 +#!/bin/bash
  2 +
  3 +BASEDIR=$(dirname $0)
  4 +
  5 +if [ $# != '1' ];
  6 +then
  7 + echo "usage : sh run.sh <scalaClassPath>"
  8 + echo "ex) sh run.sh com.piki_ds.score.TimeHyperParam"
  9 + exit
  10 +fi
  11 +
  12 +HOME=$BASEDIR
  13 +LOG_DIR=$HOME/logs
  14 +mkdir -p $LOG_DIR
  15 +TODAY=`date +"%Y%m%d"`
  16 +DATE_SUB="30"
  17 +DAY_TO_DELETE="$(date "+%Y%m%d" -d "$DATE_SUB days ago")"
  18 +DELETE_LOG="${LOG_DIR}/$1_$DAY_TO_DELETE.log"
  19 +LOG="${LOG_DIR}/$1_$TODAY.log"
  20 +
  21 +#HADOOP_CONF_DIR=/etc/hadoop/conf
  22 +
  23 +/data/spark/bin/spark-submit \
  24 +--class com.piki_ds.preprocess.$1 \
  25 +--master yarn-client \
  26 +--conf "spark.default.parallelism=250" \
  27 +$BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  28 +#target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  29 +#target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  30 +#--jars "lib_managed/jars/mysql/mysql-connector-java/mysql-connector-java-5.1.36.jar" \
  31 +
  32 +
  33 +echo "END END END END END" >> $LOG
  34 +echo "END END END END END" >> $LOG
  35 +echo "END END END END END" >> $LOG
  36 +echo "END END END END END" >> $LOG
  37 +echo "END END END END END" >> $LOG
  38 +
  39 + echo "IF LOG FILE $DELETE_LOG EXISTS DELETE" >> $LOG
  40 +# Log rotation
  41 +if [ -f $DELETE_LOG ]
  42 +then
  43 + rm -f $DELETE_LOG >> $LOG 2>&1
  44 + echo "$DELETE_LOG deleted" >> $LOG
  45 +fi