셸 스 크 립 트 사례 (2): HDFS 파일 정기 정리

#!/bin/bash
#   dfs    ,      

day=5
limit=95

basedir=/app/applogs
logfile=/app/scripts/logs/clear_hdfs_logs.log


logger(){
  msg=$1
  cdate=$(date +%F)
  ctime=$(date '+%F %T')
  echo "$ctime  $1" >> ${logfile}_$cdate
}

find_logs(){
  n=$1

  # find year
  lasted_year=$(hdfs dfs -ls $basedir |awk -F'/' 'NR>1{print $NF}' |sort -n|head -n1)
  logger "lasted year: $lasted_year"

  # find month
  basedir_year=$basedir/$lasted_year
  lasted_month=$(hdfs dfs -ls $basedir_year |awk -F'/' 'NR>1{print $NF}' |sort -n|head -n1)
  logger "lasted_month: $lasted_month"

  # find day
  basedir_month=$basedir_year/$lasted_month
  lasted_days=$(hdfs dfs -ls $basedir_month |awk -F'/' 'NR>1{print $NF}' |sort -n|head -n$n)

  for i in $lasted_days
    do 
      rm_logs $basedir_month/$i
    done 

  # check empty dir, and delete 
  n1=$(hdfs dfs -ls $basedir_month |wc -l)
  if [ $n1 -eq 0 ];then
    rm_empty $basedir_month
  fi

  n2=$(hdfs dfs -ls $basedir_year |wc -l)
    if [ $n2 -eq 0 ];then
    rm_empty $basedir_year
  fi
}

rm_logs() {
  target=$1
  logger "clear old logs: hdfs dfs -rm -r -skipTrash $target"
  hdfs dfs -rm -r -skipTrash $target
}

rm_empty() {
  target=$1
  logger "clear empty dir: hdfs dfs -rm -r -skipTrash $target"
  hdfs dfs -rm -r -skipTrash $target
}


disk_used=$(hdfs dfsadmin -report|awk -F':|%' 'NR==5 {print $3}')

if [ $(echo "$disk_used > $limit"|bc) -eq 1 ];then
  logger "current: $disk_used %, $limit +"

  find_logs $day
else
  logger "current: $disk_used %, $limit -, exit "
fi