Ale*_*lds 12 python profile performance profiling cprofile
我想提高Python脚本的性能,并一直cProfile用于生成性能报告:
python -m cProfile -o chrX.prof ./bgchr.py ...args...
我chrX.prof用Python 打开了这个文件pstats并打印出统计信息:
Python 2.7 (r27:82500, Oct 5 2010, 00:24:22)
[GCC 4.1.2 20080704 (Red Hat 4.1.2-44)] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> import pstats
>>> p = pstats.Stats('chrX.prof')
>>> p.sort_stats('name')
>>> p.print_stats()
Sun Oct 10 00:37:30 2010 chrX.prof
8760583 function calls in 13.780 CPU seconds
Ordered by: function name
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 0.000 0.000 {_locale.setlocale}
1 1.128 1.128 1.128 1.128 {bz2.decompress}
1 0.002 0.002 13.780 13.780 {execfile}
1750678 0.300 0.000 0.300 0.000 {len}
48 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects}
1 0.000 0.000 0.000 0.000 {method 'close' of 'file' objects}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
1750676 0.496 0.000 0.496 0.000 {method 'join' of 'str' objects}
1 0.007 0.007 0.007 0.007 {method 'read' of 'file' objects}
1 0.000 0.000 0.000 0.000 {method 'readlines' of 'file' objects}
1 0.034 0.034 0.034 0.034 {method 'rstrip' of 'str' objects}
23 0.000 0.000 0.000 0.000 {method 'seek' of 'file' objects}
1757785 1.230 0.000 1.230 0.000 {method 'split' of 'str' objects}
1 0.000 0.000 0.000 0.000 {method 'startswith' of 'str' objects}
1750676 0.872 0.000 0.872 0.000 {method 'write' of 'file' objects}
1 0.007 0.007 13.778 13.778 ./bgchr:3(<module>)
1 0.000 0.000 13.780 13.780 <string>:1(<module>)
1 0.001 0.001 0.001 0.001 {open}
1 0.000 0.000 0.000 0.000 {sys.exit}
1 0.000 0.000 0.000 0.000 ./bgchr:36(checkCommandLineInputs)
1 0.000 0.000 0.000 0.000 ./bgchr:27(checkInstallation)
1 1.131 1.131 13.701 13.701 ./bgchr:97(extractData)
1 0.003 0.003 0.007 0.007 ./bgchr:55(extractMetadata)
1 0.064 0.064 13.771 13.771 ./bgchr:5(main)
1750677 8.504 0.000 11.196 0.000 ./bgchr:122(parseJarchLine)
1 0.000 0.000 0.000 0.000 ./bgchr:72(parseMetadata)
1 0.000 0.000 0.000 0.000 /home/areynolds/proj/tools/lib/python2.7/locale.py:517(setlocale)
问题:我可以做些什么join,split以及write减少它们对此脚本性能的明显影响的操作？
如果它是相关的,这里是相关脚本的完整源代码:
#!/usr/bin/env python
import sys, os, time, bz2, locale
def main(*args):
# Constants
global metadataRequiredFileSize
metadataRequiredFileSize = 8192
requiredVersion = (2,5)
# Prep
global whichChromosome
whichChromosome = "all"
checkInstallation(requiredVersion)
checkCommandLineInputs()
extractMetadata()
parseMetadata()
if whichChromosome == "--list":
listMetadata()
sys.exit(0)
# Extract
extractData()
return 0
def checkInstallation(rv):
currentVersion = sys.version_info
if currentVersion[0] == rv[0] and currentVersion[1] >= rv[1]:
pass
else:
sys.stderr.write( "\n\t[%s] - Error: Your Python interpreter must be %d.%d or greater (within major version %d)\n" % (sys.argv[0], rv[0], rv[1], rv[0]) )
sys.exit(-1)
return
def checkCommandLineInputs():
cmdName = sys.argv[0]
argvLength = len(sys.argv[1:])
if (argvLength == 0) or (argvLength > 2):
sys.stderr.write( "\n\t[%s] - Usage: %s [<chromosome> | --list] <bjarch-file>\n\n" % (cmdName, cmdName) )
sys.exit(-1)
else:
global inFile
global whichChromosome
if argvLength == 1:
inFile = sys.argv[1]
elif argvLength == 2:
whichChromosome = sys.argv[1]
inFile = sys.argv[2]
if inFile == "-" or inFile == "--list":
sys.stderr.write( "\n\t[%s] - Usage: %s [<chromosome> | --list] <bjarch-file>\n\n" % (cmdName, cmdName) )
sys.exit(-1)
return
def extractMetadata():
global metadataList
global dataHandle
metadataList = []
dataHandle = open(inFile, 'rb')
try:
for data in dataHandle.readlines(metadataRequiredFileSize):
metadataLine = data
metadataLines = metadataLine.split('\n')
for line in metadataLines:
if line:
metadataList.append(line)
except IOError:
sys.stderr.write( "\n\t[%s] - Error: Could not extract metadata from %s\n\n" % (sys.argv[0], inFile) )
sys.exit(-1)
return
def parseMetadata():
global metadataList
global metadata
metadata = []
if not metadataList: # equivalent to "if len(metadataList) > 0"
sys.stderr.write( "\n\t[%s] - Error: No metadata in %s\n\n" % (sys.argv[0], inFile) )
sys.exit(-1)
for entryText in metadataList:
if entryText: # equivalent to "if len(entryText) > 0"
entry = entryText.split('\t')
filename = entry[0]
chromosome = entry[0].split('.')[0]
size = entry[1]
entryDict = { 'chromosome':chromosome, 'filename':filename, 'size':size }
metadata.append(entryDict)
return
def listMetadata():
for index in metadata:
chromosome = index['chromosome']
filename = index['filename']
size = long(index['size'])
sys.stdout.write( "%s\t%s\t%ld" % (chromosome, filename, size) )
return
def extractData():
global dataHandle
global pLength
global lastEnd
locale.setlocale(locale.LC_ALL, 'POSIX')
dataHandle.seek(metadataRequiredFileSize, 0) # move cursor past metadata
for index in metadata:
chromosome = index['chromosome']
size = long(index['size'])
pLength = 0L
lastEnd = ""
if whichChromosome == "all" or whichChromosome == index['chromosome']:
dataStream = dataHandle.read(size)
uncompressedData = bz2.decompress(dataStream)
lines = uncompressedData.rstrip().split('\n')
for line in lines:
parseJarchLine(chromosome, line)
if whichChromosome == chromosome:
break
else:
dataHandle.seek(size, 1) # move cursor past chromosome chunk
dataHandle.close()
return
def parseJarchLine(chromosome, line):
global pLength
global lastEnd
elements = line.split('\t')
if len(elements) > 1:
if lastEnd:
start = long(lastEnd) + long(elements[0])
lastEnd = long(start + pLength)
sys.stdout.write("%s\t%ld\t%ld\t%s\n" % (chromosome, start, lastEnd, '\t'.join(elements[1:])))
else:
lastEnd = long(elements[0]) + long(pLength)
sys.stdout.write("%s\t%ld\t%ld\t%s\n" % (chromosome, long(elements[0]), lastEnd, '\t'.join(elements[1:])))
else:
if elements[0].startswith('p'):
pLength = long(elements[0][1:])
else:
start = long(long(lastEnd) + long(elements[0]))
lastEnd = long(start + pLength)
sys.stdout.write("%s\t%ld\t%ld\n" % (chromosome, start, lastEnd))
return
if __name__ == '__main__':
sys.exit(main(*sys.argv))
编辑
如果我sys.stdout.write在第一个条件中注释掉语句parseJarchLine(),那么我的运行时间从10.2秒到4.8秒:
# with first conditional's "sys.stdout.write" enabled
$ time ./bgchr chrX test.bjarch > /dev/null
real 0m10.186s
user 0m9.917s
sys 0m0.160s
# after first conditional's "sys.stdout.write" is commented out
$ time ./bgchr chrX test.bjarch > /dev/null
real 0m4.808s
user 0m4.561s
sys 0m0.156s
写stdout在Python中真的那么贵吗？
Joh*_*hin 27
ncalls仅与将数字与其他计数(如文件中的字符/字段/行数)进行比较可能会导致高度异常的程度相关; tottime而cumtime这才是真正重要的.cumtime是在函数/方法中花费的时间,包括在它调用的函数/方法中花费的时间; tottime是在函数/方法中花费的时间,不包括它调用的函数/方法所花费的时间.
我发现对统计数据进行排序tottime和重新排序很有帮助cumtime,而不是name.
bgchar definitely refers to the execution of the script and is not irrelevant as it takes up 8.9 seconds out of 13.5; that 8.9 seconds does NOT include time in the functions/methods that it calls! Read carefully what @Lie Ryan says about modularising your script into functions, and implement his advice. Likewise what @jonesy says.
string之所以提到是因为你import string只在一个地方使用它: string.find(elements[0], 'p').在输出的另一行,您会注意到string.find只被调用一次,因此在此脚本的运行中不是性能问题.但是:你str在其他地方使用方法.string函数现在已弃用,并通过调用相应的str方法来实现.你会更好地写elements[0].find('p') == 0一个确切但更快的等价物,并且可能想用elements[0].startswith('p')它来节省读者,想知道这== 0是否真的应该存在 == -1.
@Bernd Petersohn提到的四种方法仅占用了总执行时间13.541秒的3.7秒.在过度担心这些问题之前,先将脚本模块化为函数,再次运行cProfile,然后对其进行排序tottime.
使用更改后的脚本修改问题后更新:
msgstr"""问题:我可以做些什么来加入,拆分和写入操作,以减少它们对此脚本性能的明显影响？""
咦？那些3个在总共13.8个中占2.6秒.您的parseJarchLine函数需要8.5秒(不包括它调用的函数/方法所花费的时间).assert(8.5 > 2.6)
Bernd已经指出了你可能会考虑做些什么.你不必要地完全拆分线路,只有在写出来时再次连接它.您只需要检查第一个元素.而不是elements = line.split('\t')做elements = line.split('\t', 1)和替换'\t'.join(elements[1:])的elements[1].
现在让我们深入了解parseJarchLine的主体.源的使用次数和long内置函数的使用方式令人惊讶.同样令人惊讶的是longcProfile输出中没有提到的事实.
为什么你需要long？超过2 Gb的文件？好的,那么你需要考虑自Python 2.2以来,int溢出导致升级long而不是引发异常.您可以利用更快的int算术执行.您还需要考虑做什么long(x)时候x已经证明long是浪费资源.
这是parseJarchLine函数,其中删除 - 废弃更改标记为[1],更改为int更改标记为[2].好主意:以小步骤进行更改,重新测试,重新分析.
def parseJarchLine(chromosome, line):
global pLength
global lastEnd
elements = line.split('\t')
if len(elements) > 1:
if lastEnd != "":
start = long(lastEnd) + long(elements[0])
# [1] start = lastEnd + long(elements[0])
# [2] start = lastEnd + int(elements[0])
lastEnd = long(start + pLength)
# [1] lastEnd = start + pLength
sys.stdout.write("%s\t%ld\t%ld\t%s\n" % (chromosome, start, lastEnd, '\t'.join(elements[1:])))
else:
lastEnd = long(elements[0]) + long(pLength)
# [1] lastEnd = long(elements[0]) + pLength
# [2] lastEnd = int(elements[0]) + pLength
sys.stdout.write("%s\t%ld\t%ld\t%s\n" % (chromosome, long(elements[0]), lastEnd, '\t'.join(elements[1:])))
else:
if elements[0].startswith('p'):
pLength = long(elements[0][1:])
# [2] pLength = int(elements[0][1:])
else:
start = long(long(lastEnd) + long(elements[0]))
# [1] start = lastEnd + long(elements[0])
# [2] start = lastEnd + int(elements[0])
lastEnd = long(start + pLength)
# [1] lastEnd = start + pLength
sys.stdout.write("%s\t%ld\t%ld\n" % (chromosome, start, lastEnd))
return
问题后更新 sys.stdout.write
如果您注释掉的声明与原始声明完全相同:
sys.stdout.write("%s\t%ld\t%ld\t%s\n" % (chromosome, start, lastEnd, '\t'.join(elements[1:])))
然后你的问题是......有趣.试试这个:
payload = "%s\t%ld\t%ld\t%s\n" % (chromosome, start, lastEnd, '\t'.join(elements[1:]))
sys.stdout.write(payload)
现在评论sys.stdout.write声明......
顺便说一下,有人在评论中提到将这个打成多个写入...你有没有考虑过这个？元素[1:]中平均有多少字节？在染色体？
===话题的变化:我很担心,你初始化lastEnd到"",而不是零,而且没有人发表了评论.无论如何,你应该解决这个问题,这样可以进行相当大的简化,并添加其他人的建议:
def parseJarchLine(chromosome, line):
global pLength
global lastEnd
elements = line.split('\t', 1)
if elements[0][0] == 'p':
pLength = int(elements[0][1:])
return
start = lastEnd + int(elements[0])
lastEnd = start + pLength
sys.stdout.write("%s\t%ld\t%ld" % (chromosome, start, lastEnd))
if elements[1:]:
sys.stdout.write(elements[1])
sys.stdout.write(\n)
现在我同样担心两个全局变量lastEnd和pLength-现在parseJarchLine功能是如此之小,它可以折叠回到其唯一的调用者的身体extractData,从而节省了两个全局变量,一个极大的函数调用.您还可以sys.stdout.write通过将write = sys.stdout.write前一次放在前面extractData并使用它来保存大量的查找.
顺便说一句,脚本测试Python 2.5或更高版本; 你有没有试过2.5和2.6的分析？