shell gist to get http 5XX code from ELB access log

一个简陋的只是够用的shell小脚本，用来从S3中下载前一天的ELB access log，筛选出http code为5XX的日志，然后发送给某个指定的email地址。

变量设置

设置下MAIL_TO和S3_BASE_PREFIX这两个变量，然后在有AWS CLI访问环境的机器上执行即可。

  • MAIL_TO 如果有多个地址，可用空格隔开
  • S3_BASE_PREFIX设置到Access log日志格式之前的prefix就行。

比如完整的access log地址是: s3://carl-elb-logs/AWSLogs/888888888888/elasticloadbalancing/ap-northeast-2/2017/12/24/704017765382_elasticloadbalancing_ap-northeast-2_ELB-carl_20171224T0120Z_13.125.86.136_5s9i5ef2.log

那么S3_BASE_PREFIX设置为s3://carl-elb-logs/AWSLogs/888888888888/elasticloadbalancing/ap-northeast-2即可

代码

Gist地址: https://gist.github.com/jibing57/fd241ab78d0243252a4b19ba19f69fe8

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
#!/bin/bash

#############################################
###### Global Variable  #########
#############################################

MAIL_TO=""
S3_BASE_PREFIX=""

BASE_WORK_DIR=`cd $(dirname $0); pwd `

#############################################
###### Common Function #########
#############################################

function log()
{
    timer=`date "+%Y-%m-%d %H:%M:%S"`
    echo "$timer -- $1"
}

function err_log()
{
    log "[Error]: $1"
}

function create_dir()
{
    dir_name=$1

    if [ ! -d $dir_name ]; then
        log "dir [$dir_name] is not existed, should create it"
        mkdir -p $dir_name
    fi

    if [ ! -d $dir_name ]; then
        err_log "dir [$dir_name] is not existed, and can't create it"
        exit -1
    fi

    return 0
}

#############################################
###### Main Process #########
#############################################

if [ "$MAIL_TO" == "" ]; then
    log "Error: MAIL_TO is empty"
    exit 1
fi

if [ "$S3_BASE_PREFIX" == "" ]; then
    log "Error: S3_BASE_PREFIX is empty"
    exit 1
fi

is_mac=false
os_name=$(uname -s)

if [[ "$os_name" == "Linux" ]]; then
    #statements
    is_mac=false
elif [[ "$os_name" == "Darwin" ]]; then
    is_mac=true
fi

if [[ $is_mac == true ]]; then
    YEAR=`date -v -1d +%Y`
    MONTH=`date -v -1d +%m`
    DAY=`date -v -1d +%d`
else
    YEAR=`date +"%Y" -d "1 day ago"`
    MONTH=`date +"%m" -d "1 day ago"`
    DAY=`date +"%d" -d "1 day ago"`
fi

log "=== Try to process access log of ELB on YEAR=$YEAR, MONTH=$MONTH, DAY=$DAY"

DATE_PATH="$YEAR/$MONTH/$DAY"

cd $BASE_WORK_DIR
RESULT_DIR=$BASE_WORK_DIR/result/
WORK_DIR="$BASE_WORK_DIR/$DATE_PATH/"
log "=== try to create dir $WORK_DIR"
create_dir $WORK_DIR
log "=== try to create dir $RESULT_DIR"
create_dir $RESULT_DIR
cd $WORK_DIR
log "=== process work dir is $WORK_DIR, current dir is `pwd`"

log "===== Download s3 file start ======="
# download s3 file to local dir
log "aws s3 sync $S3_BASE_PREFIX/$DATE_PATH/ ./"
aws s3 sync $S3_BASE_PREFIX/$DATE_PATH/ ./
log "===== Download s3 file end ======="

log "===== grep 5XX start ======="
DATE_LOG_NAME="${YEAR}_${MONTH}_${DAY}.log"
RESULT_FILE="$RESULT_DIR/${DATE_LOG_NAME}"
grep "5[0-9]\{2\} 5[0-9]\{2\}" *.log > $RESULT_FILE
cd $BASE_WORK_DIR
rm -r $WORK_DIR/*.log
log "===== grep 5XX end ======="

log "=== File on result_dir[${RESULT_DIR}] is $(ls ${RESULT_DIR}/*${DATE_LOG_NAME})"

#############################################
###### Send Mail #########
#############################################

subject="Report of Http 5XX in ELB Access log on ${YEAR}/${MONTH}/${DAY}"
MAIL_ATTACHED_OPTIONS=""

for result_file in `ls ${RESULT_DIR}/${DATE_LOG_NAME}`
do
    MAIL_ATTACHED_OPTIONS=" ${MAIL_ATTACHED_OPTIONS} -a ${result_file}"
done

log "MAIL_ATTACHED_OPTIONS is [${MAIL_ATTACHED_OPTIONS}]"

log "=== send report to email $MAIL_TO"

if [[ $is_mac == true ]]; then
    echo -e "Attached is the http 5XX log on ${YEAR}/${MONTH}/${DAY}" | mail -s "${subject}" $MAIL_TO
else
    echo -e "Attached is the http 5XX log on ${YEAR}/${MONTH}/${DAY}" | mail -s "${subject}" $MAIL_ATTACHED_OPTIONS $MAIL_TO
fi

延伸阅读

留言

▲