Blame view

mapwithrank.sh 1.14 KB
333e10412   Joanne   adding shells
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
  #!/bin/bash
  
  BASEDIR=$(dirname $0)
  
  HOME=$BASEDIR
  LOG_DIR=$HOME/logs
  mkdir -p $LOG_DIR
  TODAY=`date +"%Y%m%d"`
  DATE_SUB="10"
  DAY_TO_DELETE="$(date "+%Y%m%d" -d "$DATE_SUB days ago")"
  DELETE_LOG="${LOG_DIR}/MapWithRank_$DAY_TO_DELETE.log"
  LOG="${LOG_DIR}/MapWithRank_$TODAY.log"
  
  #HADOOP_CONF_DIR=/etc/hadoop/conf
  
  /data/spark/bin/spark-submit \
  --class com.piki_ds.preprocess.MapWithRank \
  --master yarn-client \
d89affd59   aram   map with rank con...
19
20
  --executor-memory 4000m \
  --driver-memory 5000m \
333e10412   Joanne   adding shells
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
  $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
  #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar  >> $LOG 2>&1
  #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar  >> $LOG 2>&1
  #--jars "lib_managed/jars/mysql/mysql-connector-java/mysql-connector-java-5.1.36.jar" \
  
  
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  echo "END END END END END" >> $LOG
  
   echo "IF LOG FILE $DELETE_LOG EXISTS DELETE" >> $LOG
  # Log rotation
  if [ -f $DELETE_LOG ]
  then
      rm -f $DELETE_LOG >> $LOG 2>&1
      echo "$DELETE_LOG deleted" >> $LOG
  fi
  #!/usr/bin/env bash#!/usr/bin/env bash