Python实现的大数据分析操作系统日志功能示例_Python

Python实现的大数据分析操作系统日志功能示例

2021-05-27 00:11chengqiuming Python

这篇文章主要介绍了Python实现的大数据分析操作系统日志功能,涉及Python大文件切分、读取、多线程操作等相关使用技巧,需要的朋友可以参考下

本文实例讲述了Python实现的大数据分析操作系统日志功能。分享给大家供大家参考，具体如下：

一代码

1、大文件切分

				?

									import os

									import os.path

									import time

									def FileSplit(sourceFile, targetFolder):

									  if not os.path.isfile(sourceFile):

									    print(sourceFile, ' does not exist.')

									    return

									  if not os.path.isdir(targetFolder):

									    os.mkdir(targetFolder)

									  tempData = []

									  number = 1000

									  fileNum = 1

									  linesRead = 0

									  with open(sourceFile, 'r') as srcFile:

									    dataLine = srcFile.readline().strip()

									    while dataLine:

									      for i in range(number):

									        tempData.append(dataLine)

									        dataLine = srcFile.readline()

									        if not dataLine:

									          break

									      desFile = os.path.join(targetFolder, sourceFile[0:-4] + str(fileNum) + '.txt')

									      with open(desFile, 'a+') as f:

									        f.writelines(tempData)

									      tempData = []

									      fileNum = fileNum + 1

									if __name__ == '__main__':

									  #sourceFile = input('Input the source file to split:')

									  #targetFolder = input('Input the target folder you want to place the split files:')

									  sourceFile = 'test.txt'

									  targetFolder = 'test'

									  FileSplit(sourceFile, targetFolder)

2、Mapper代码

				?

									import os

									import re

									import threading

									import time

									def Map(sourceFile):

									  if not os.path.exists(sourceFile):

									    print(sourceFile, ' does not exist.')

									    return

									  pattern = re.compile(r'[0-9]{1,2}/[0-9]{1,2}/[0-9]{4}')

									  result = {}

									  with open(sourceFile, 'r') as srcFile:

									    for dataLine in srcFile:

									      r = pattern.findall(dataLine)

									      if r:

									        t = result.get(r[0], 0)

									        t += 1

									        result[r[0]] = t

									  desFile = sourceFile[0:-4] + '_map.txt'

									  with open(desFile, 'a+') as fp:

									    for k, v in result.items():

									      fp.write(k + ':' + str(v) + '\n')

									if __name__ == '__main__':

									  desFolder = 'test'

									  files = os.listdir(desFolder)

									  #如果不使用多线程，可以直接这样写

									  '''for f in files:

									    Map(desFolder + '\\' + f)'''

									  #使用多线程

									  def Main(i):

									    Map(desFolder + '\\' + files[i])

									  fileNumber = len(files)

									  for i in range(fileNumber):

									    t = threading.Thread(target = Main, args =(i,))

									    t.start()

3.Reducer代码

				?

									import os

									def Reduce(sourceFolder, targetFile):

									  if not os.path.isdir(sourceFolder):

									    print(sourceFolder, ' does not exist.')

									    return

									  result = {}

									  #Deal only with the mapped files

									  allFiles = [sourceFolder+'\\'+f for f in os.listdir(sourceFolder) if f.endswith('_map.txt')]

									  for f in allFiles:

									    with open(f, 'r') as fp:

									      for line in fp:

									        line = line.strip()

									        if not line:

									          continue

									        position = line.index(':')

									        key = line[0:position]

									        value = int(line[position + 1:])

									        result[key] = result.get(key,0) + value

									  with open(targetFile, 'w') as fp:

									    for k,v in result.items():

									      fp.write(k + ':' + str(v) + '\n')

									if __name__ == '__main__':

									  Reduce('test', 'test\\result.txt')