data science team / dsquality

Download as
Browse Code ยป

Commit 333e1041247a6bb1aea0ca06082d391dbea948ba

Authored by Joanne ago
1 parent 2a033a39da
Exists in master

adding shells

Showing 3 changed files with 77 additions and 1 deletions Side-by-side Diff

ggh_related.sh View file @ 333e104
1 1 #!/bin/bash
2 2  
3 3 BASEDIR=$(dirname $0)
4   -sh $BASEDIR/preprocess.sh MakingLapse
  4 +sh $BASEDIR/makelapse.sh
5 5 sh $BASEDIR/preprocess.sh MapWithRank
6 6  
7 7 sh $BASEDIR/combinedRun.sh content
makelapse.sh View file @ 333e104
  1 +#!/bin/bash
  2 +
  3 +BASEDIR=$(dirname $0)
  4 +
  5 +HOME=$BASEDIR
  6 +LOG_DIR=$HOME/logs
  7 +mkdir -p $LOG_DIR
  8 +TODAY=`date +"%Y%m%d"`
  9 +DATE_SUB="10"
  10 +DAY_TO_DELETE="$(date "+%Y%m%d" -d "$DATE_SUB days ago")"
  11 +DELETE_LOG="${LOG_DIR}/MakingLapse_$DAY_TO_DELETE.log"
  12 +LOG="${LOG_DIR}/MakingLapse_$TODAY.log"
  13 +
  14 +#HADOOP_CONF_DIR=/etc/hadoop/conf
  15 +
  16 +/data/spark/bin/spark-submit \
  17 +--class com.piki_ds.preprocess.MakingLapse \
  18 +--master yarn-client \
  19 +$BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  20 +#target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  21 +#target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  22 +#--jars "lib_managed/jars/mysql/mysql-connector-java/mysql-connector-java-5.1.36.jar" \
  23 +
  24 +
  25 +echo "END END END END END" >> $LOG
  26 +echo "END END END END END" >> $LOG
  27 +echo "END END END END END" >> $LOG
  28 +echo "END END END END END" >> $LOG
  29 +echo "END END END END END" >> $LOG
  30 +
  31 + echo "IF LOG FILE $DELETE_LOG EXISTS DELETE" >> $LOG
  32 +# Log rotation
  33 +if [ -f $DELETE_LOG ]
  34 +then
  35 + rm -f $DELETE_LOG >> $LOG 2>&1
  36 + echo "$DELETE_LOG deleted" >> $LOG
  37 +fi
  38 +#!/usr/bin/env bash
mapwithrank.sh View file @ 333e104
  1 +#!/bin/bash
  2 +
  3 +BASEDIR=$(dirname $0)
  4 +
  5 +HOME=$BASEDIR
  6 +LOG_DIR=$HOME/logs
  7 +mkdir -p $LOG_DIR
  8 +TODAY=`date +"%Y%m%d"`
  9 +DATE_SUB="10"
  10 +DAY_TO_DELETE="$(date "+%Y%m%d" -d "$DATE_SUB days ago")"
  11 +DELETE_LOG="${LOG_DIR}/MapWithRank_$DAY_TO_DELETE.log"
  12 +LOG="${LOG_DIR}/MapWithRank_$TODAY.log"
  13 +
  14 +#HADOOP_CONF_DIR=/etc/hadoop/conf
  15 +
  16 +/data/spark/bin/spark-submit \
  17 +--class com.piki_ds.preprocess.MapWithRank \
  18 +--master yarn-client \
  19 +$BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  20 +#target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  21 +#target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  22 +#--jars "lib_managed/jars/mysql/mysql-connector-java/mysql-connector-java-5.1.36.jar" \
  23 +
  24 +
  25 +echo "END END END END END" >> $LOG
  26 +echo "END END END END END" >> $LOG
  27 +echo "END END END END END" >> $LOG
  28 +echo "END END END END END" >> $LOG
  29 +echo "END END END END END" >> $LOG
  30 +
  31 + echo "IF LOG FILE $DELETE_LOG EXISTS DELETE" >> $LOG
  32 +# Log rotation
  33 +if [ -f $DELETE_LOG ]
  34 +then
  35 + rm -f $DELETE_LOG >> $LOG 2>&1
  36 + echo "$DELETE_LOG deleted" >> $LOG
  37 +fi
  38 +#!/usr/bin/env bash#!/usr/bin/env bash