基于Python的关键字监控及告警_Python

为了解决日志文件监控的问题，使用python脚本完成了基于关键字的告警功能

环境 python 2.7

依赖包 time \ traceback \ filelock \ logging

代码如下：

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

									#!/bin/python

									#coding:utf-8

									import sys

									reload(sys)

									sys.setdefaultencoding('utf8')

									import re

									import os

									from urllib import urlencode

									import logging

									import filelock

									import time

									import traceback

									#config.conf

									#文件1:关键字A|关键字B:出现次数:告警方式:联系方式:联系组:某某异常

									#文件2:关键字C|关键字D:出现次数:告警方式:联系方式:联系组:某某异常

									#rc.local增加

									#sudo -u monitor /bin/bash -x /home/apps/logmon-job/deploy_py.sh

									logging.basicConfig(level=logging.DEBUG,

									          format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',

									          datefmt='%a, %d %b %Y %H:%M:%S',

									          filename='/home/logs/monitor/logmon.log',

									          #filename='/Users/mac/Desktop/logmon/logmon.log',

									          filemode='a')

									basDir='/home/apps/logmon-job/'

									posFiles=basDir+'/pos.log'

									configFile=basDir+'config.conf'

									def readOnly(filename):

									  return open(filename,'r')

									  # pass

									def readWrite(filename):

									  return open(filename,'rw')

									  # pass

									def writeOnly(filename):

									  return open(filename,'w')

									  # pass

									# def closesfile():

									#   pass

									def getStartPosLog(posFiles):

									  txt=readOnly(posFiles)

									  result={}

									  for i in txt :

									    filename,pos=i.split(':')

									    if filename != '' :

									      result[filename]=pos

									  return result

									  txt.close()

									def rematch(txt,regular):

									  resultList=[]

									  for t in txt.split(r'\n') :

									    # print t

									    # pattern = re.compile(r':')

									    pattern = re.compile(regular)

									    resultList=(pattern.findall(t))

									  try :

									    # print '匹配结果为',resultList 

									    return len(resultList),regular , resultList[0]

									  except Exception as e :

									    print e 

									    return 0 , regular , ''

									  # pass

									def getEndPost(f):

									  filename=readOnly(f)

									  try :

									    nowpos=filename.tell()

									    filename.seek(0,2)

									    endpos=filename.tell()

									    filename.seek(nowpos,0)

									  except :

									    endpos = 0

									  filename.close()

									  return endpos 

									  # pass

									def getDistinct(startpos,endpos):

									  return endpos-startpos

									def getText(f,startpos,endpos):

									  filename=readOnly(f)

									  filename.seek(startpos,0)

									  textLength=getDistinct(startpos,endpos)

									  text=filename.read(textLength)

									  filename.close()

									  return text

									def updatePosLog(posResult,posFiles):

									  f=writeOnly(posFiles)

									  # print 'posResult ',posResult

									  for k in posResult.keys() :

									    v=posResult[k]

									    f.writelines('%s:%s\n' %(k,v))

									  f.close()

									  pass

									def getAlterConfi(filename):

									  #文件:关键字:出现次数:告警方式:联系方式:联系组

									  f=readOnly(filename)

									  result={}

									  for lines in f.readlines():

									    # print lines

									    try :

									      filename , key , count , alterType , alterAddress , alterGroup ,alterMessage= lines.strip('\n').split(":")

									      result[filename]={}

									      result[filename]["key"]     =key

									      result[filename]["count"]    =count

									      result[filename]["alterType"]  =alterType

									      result[filename]["alterAddress"]=alterAddress

									      result[filename]["alterGroup"] =alterGroup

									      result[filename]["alterMessage"]=alterMessage

									    except Exception as e:

									      print e

									      print '错误的配置 %s' % (lines.strip('\n'))

									      pass

									  return result

									def sendSms(account,message):

									  data={

									    'accounts':account ,

									    'templateName':'opalert' ,

									    'alertcontent':message ,

									  }

									  encodeMessage=urlencode(data)

									  #正式时需要开启

									  os.system('curl -I "http://10.1.1.146:8080/sms/send?%s" ' % ( encodeMessage ) )

									def main():

									  global posFiles

									  global configFile

									  AlterConfi=getAlterConfi(configFile)

									  print AlterConfi

									  posResult=getStartPosLog(posFiles)

									  posResult_bak=getStartPosLog(posFiles)

									  # print posResult

									  for filename in AlterConfi.keys() :

									    keyDict=AlterConfi[filename]

									    print '开始检查文件 ',filename

									    #print rematch(filename,r'#')[0] 

									    if not os.path.exists(filename):

									      print 'file "%s" not exist ,pass' % (filename)

									      # continue

									    if os.path.exists(filename):

									      endpos = getEndPost(filename)

									      if endpos == 0 :

									        print 'file "%s" is empty ,pass' % (filename)

									      else :

									        try :

									          startpos= int(posResult[filename])

									        except :

									          startpos = 0

									        print 'startpos is %.f , endpos is %.f' %(startpos ,endpos)

									        #处理切割后，偏移量归位

									        if startpos > endpos :

									          startpos = 0

									        text = getText(filename,startpos,endpos)

									        # print '%s text is : '%(filename) , text

									        #分析关键字

									        #print AposlterConfi[filename]

									        matchCount , regular , resultList = rematch(text,keyDict['key'])

									        print '匹配关键字',regular , '匹配长度为', matchCount , '关键字告警阈值' ,keyDict['count'] , '关键字' , resultList

									        if int(matchCount) >= int(keyDict['count']) :

									          print 'alterGroup len is ',len(keyDict['alterGroup'])

									          print 'alterType len is ' ,len(keyDict['alterType'])

									          if len(keyDict['alterGroup']) > 0:

									            pass

									          if len(keyDict['alterType']) >0:

									            if keyDict['alterType'].upper() == 'SMS' :

									              for account in keyDict['alterAddress'].split(',') :

									                if len(account) >0 :

									                  sendSms(account,'发现%s 告警，关键字:%s ,出现次数:%s ' %(keyDict['alterMessage'] , resultList , matchCount ))

									            pass

									        #记录末尾偏移量

									        posResult_bak[filename]=endpos

									  print '打印文件偏移量信息',posResult_bak

									  #正式时需要开启

									  updatePosLog(posResult_bak,posFiles)

									if __name__ == '__main__':

									  lock = filelock.FileLock("/home/apps/logmon-job/logmon.py.lock")

									  if lock:

									    logging.info("CaiWeiCheng Get Lock.start!!!")

									  try:

									    with lock.acquire(timeout=5):

									      while 1 :

									        main()

									        time.sleep(60)

									      # pass

									  #except filelock.timeout :

									  except Exception as e :

									    print traceback.format_exc()

									    print "timeout"    

									    logging.warning("get file lock timeout")