got*_*nes 80 python concurrency multiprocessing embarrassingly-parallel
令人尴尬的并行问题通常包括三个基本部分:
我们可以在两个方面并行化程序:
这似乎是并发编程中最基本的模式,但我仍然试图解决它,所以让我们写一个规范的例子来说明如何使用多处理来完成.
下面是示例问题:给定一个包含整数行作为输入的CSV文件,计算它们的总和.将问题分成三个部分,这些部分可以并行运行:
下面是传统的单进程绑定Python程序,它解决了以下三个任务:
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# basicsums.py
"""A program that reads integer values from a CSV file and writes out their
sums to another CSV file.
"""
import csv
import optparse
import sys
def make_cli_parser():
"""Make the command line interface parser."""
usage = "\n\n".join(["python %prog INPUT_CSV OUTPUT_CSV",
__doc__,
"""
ARGUMENTS:
INPUT_CSV: an input CSV file with rows of numbers
OUTPUT_CSV: an output file that will contain the sums\
"""])
cli_parser = optparse.OptionParser(usage)
return cli_parser
def parse_input_csv(csvfile):
"""Parses the input CSV and yields tuples with the index of the row
as the first element, and the integers of the row as the second
element.
The index is zero-index based.
:Parameters:
- `csvfile`: a `csv.reader` instance
"""
for i, row in enumerate(csvfile):
row = [int(entry) for entry in row]
yield i, row
def sum_rows(rows):
"""Yields a tuple with the index of each input list of integers
as the first element, and the sum of the list of integers as the
second element.
The index is zero-index based.
:Parameters:
- `rows`: an iterable of tuples, with the index of the original row
as the first element, and a list of integers as the second element
"""
for i, row in rows:
yield i, sum(row)
def write_results(csvfile, results):
"""Writes a series of results to an outfile, where the first column
is the index of the original row of data, and the second column is
the result of the calculation.
The index is zero-index based.
:Parameters:
- `csvfile`: a `csv.writer` instance to which to write results
- `results`: an iterable of tuples, with the index (zero-based) of
the original row as the first element, and the calculated result
from that row as the second element
"""
for result_row in results:
csvfile.writerow(result_row)
def main(argv):
cli_parser = make_cli_parser()
opts, args = cli_parser.parse_args(argv)
if len(args) != 2:
cli_parser.error("Please provide an input file and output file.")
infile = open(args[0])
in_csvfile = csv.reader(infile)
outfile = open(args[1], 'w')
out_csvfile = csv.writer(outfile)
# gets an iterable of rows that's not yet evaluated
input_rows = parse_input_csv(in_csvfile)
# sends the rows iterable to sum_rows() for results iterable, but
# still not evaluated
result_rows = sum_rows(input_rows)
# finally evaluation takes place as a chain in write_results()
write_results(out_csvfile, result_rows)
infile.close()
outfile.close()
if __name__ == '__main__':
main(sys.argv[1:])
让我们采用这个程序并重写它以使用多处理来并行化上面概述的三个部分.下面是这个新的并行化程序的框架,需要充实以解决注释中的部分:
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# multiproc_sums.py
"""A program that reads integer values from a CSV file and writes out their
sums to another CSV file, using multiple processes if desired.
"""
import csv
import multiprocessing
import optparse
import sys
NUM_PROCS = multiprocessing.cpu_count()
def make_cli_parser():
"""Make the command line interface parser."""
usage = "\n\n".join(["python %prog INPUT_CSV OUTPUT_CSV",
__doc__,
"""
ARGUMENTS:
INPUT_CSV: an input CSV file with rows of numbers
OUTPUT_CSV: an output file that will contain the sums\
"""])
cli_parser = optparse.OptionParser(usage)
cli_parser.add_option('-n', '--numprocs', type='int',
default=NUM_PROCS,
help="Number of processes to launch [DEFAULT: %default]")
return cli_parser
def main(argv):
cli_parser = make_cli_parser()
opts, args = cli_parser.parse_args(argv)
if len(args) != 2:
cli_parser.error("Please provide an input file and output file.")
infile = open(args[0])
in_csvfile = csv.reader(infile)
outfile = open(args[1], 'w')
out_csvfile = csv.writer(outfile)
# Parse the input file and add the parsed data to a queue for
# processing, possibly chunking to decrease communication between
# processes.
# Process the parsed data as soon as any (chunks) appear on the
# queue, using as many processes as allotted by the user
# (opts.numprocs); place results on a queue for output.
#
# Terminate processes when the parser stops putting data in the
# input queue.
# Write the results to disk as soon as they appear on the output
# queue.
# Ensure all child processes have terminated.
# Clean up files.
infile.close()
outfile.close()
if __name__ == '__main__':
main(sys.argv[1:])
这些代码片段以及可以生成用于测试目的的示例CSV文件的另一段代码可以在github上找到.
关于你如何让并发专家来解决这个问题,我将不胜感激.
以下是我在思考这个问题时遇到的一些问题.解决任何/所有问题的奖励积分:
hba*_*bar 67
我的解决方案有一个额外的铃声和哨子,以确保输出的顺序与输入的顺序相同.我使用multiprocessing.queue来在进程之间发送数据,发送停止消息,以便每个进程知道退出检查队列.我认为来源中的评论应该清楚说明发生了什么,但如果没有让我知道.
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# multiproc_sums.py
"""A program that reads integer values from a CSV file and writes out their
sums to another CSV file, using multiple processes if desired.
"""
import csv
import multiprocessing
import optparse
import sys
NUM_PROCS = multiprocessing.cpu_count()
def make_cli_parser():
"""Make the command line interface parser."""
usage = "\n\n".join(["python %prog INPUT_CSV OUTPUT_CSV",
__doc__,
"""
ARGUMENTS:
INPUT_CSV: an input CSV file with rows of numbers
OUTPUT_CSV: an output file that will contain the sums\
"""])
cli_parser = optparse.OptionParser(usage)
cli_parser.add_option('-n', '--numprocs', type='int',
default=NUM_PROCS,
help="Number of processes to launch [DEFAULT: %default]")
return cli_parser
class CSVWorker(object):
def __init__(self, numprocs, infile, outfile):
self.numprocs = numprocs
self.infile = open(infile)
self.outfile = outfile
self.in_csvfile = csv.reader(self.infile)
self.inq = multiprocessing.Queue()
self.outq = multiprocessing.Queue()
self.pin = multiprocessing.Process(target=self.parse_input_csv, args=())
self.pout = multiprocessing.Process(target=self.write_output_csv, args=())
self.ps = [ multiprocessing.Process(target=self.sum_row, args=())
for i in range(self.numprocs)]
self.pin.start()
self.pout.start()
for p in self.ps:
p.start()
self.pin.join()
i = 0
for p in self.ps:
p.join()
print "Done", i
i += 1
self.pout.join()
self.infile.close()
def parse_input_csv(self):
"""Parses the input CSV and yields tuples with the index of the row
as the first element, and the integers of the row as the second
element.
The index is zero-index based.
The data is then sent over inqueue for the workers to do their
thing. At the end the input process sends a 'STOP' message for each
worker.
"""
for i, row in enumerate(self.in_csvfile):
row = [ int(entry) for entry in row ]
self.inq.put( (i, row) )
for i in range(self.numprocs):
self.inq.put("STOP")
def sum_row(self):
"""
Workers. Consume inq and produce answers on outq
"""
tot = 0
for i, row in iter(self.inq.get, "STOP"):
self.outq.put( (i, sum(row)) )
self.outq.put("STOP")
def write_output_csv(self):
"""
Open outgoing csv file then start reading outq for answers
Since I chose to make sure output was synchronized to the input there
is some extra goodies to do that.
Obviously your input has the original row number so this is not
required.
"""
cur = 0
stop = 0
buffer = {}
# For some reason csv.writer works badly across processes so open/close
# and use it all in the same process or else you'll have the last
# several rows missing
outfile = open(self.outfile, "w")
self.out_csvfile = csv.writer(outfile)
#Keep running until we see numprocs STOP messages
for works in range(self.numprocs):
for i, val in iter(self.outq.get, "STOP"):
# verify rows are in order, if not save in buffer
if i != cur:
buffer[i] = val
else:
#if yes are write it out and make sure no waiting rows exist
self.out_csvfile.writerow( [i, val] )
cur += 1
while cur in buffer:
self.out_csvfile.writerow([ cur, buffer[cur] ])
del buffer[cur]
cur += 1
outfile.close()
def main(argv):
cli_parser = make_cli_parser()
opts, args = cli_parser.parse_args(argv)
if len(args) != 2:
cli_parser.error("Please provide an input file and output file.")
c = CSVWorker(opts.numprocs, args[0], args[1])
if __name__ == '__main__':
main(sys.argv[1:])
老套。
p1.py
import csv
import pickle
import sys
with open( "someFile", "rb" ) as source:
rdr = csv.reader( source )
for line in eumerate( rdr ):
pickle.dump( line, sys.stdout )
p2.py
import pickle
import sys
while True:
try:
i, row = pickle.load( sys.stdin )
except EOFError:
break
pickle.dump( i, sum(row) )
p3.py
import pickle
import sys
while True:
try:
i, row = pickle.load( sys.stdin )
except EOFError:
break
print i, row
这是多处理的最终结构。
python p1.py | python p2.py | python p3.py
是的，shell 在操作系统级别将它们编织在一起。对我来说这似乎更简单而且效果很好。
是的，使用 pickle（或 cPickle）的开销稍微多一些。然而，这种简化似乎值得付出努力。
如果您希望文件名成为 的参数p1.py，那么这是一个简单的更改。
更重要的是，像下面这样的函数非常方便。
def get_stdin():
while True:
try:
yield pickle.load( sys.stdin )
except EOFError:
return
这允许你这样做：
for item in get_stdin():
process item
这很简单，但它并不容易让您运行多个 P2.py 副本。
您有两个问题：扇出和扇入。P1.py 必须以某种方式扇出到多个 P2.py。P2.py 必须以某种方式将其结果合并到单个 P3.py 中。
老式的扇出方法是“推”架构，这是非常有效的。
理论上，多个P2.py从一个公共队列中拉取是资源的最优分配。这通常是理想的，但它也是相当大量的编程。编程真的有必要吗？或者循环处理就足够了吗？
实际上，您会发现让 P1.py 在多个 P2.py 之间进行简单的“循环”处理可能非常好。您将 P1.py 配置为通过命名管道处理P2.py 的n 个副本。P2.py 将从各自适当的管道中读取数据。
如果一个 P2.py 获取了所有“最坏情况”数据并且运行远远落后怎么办？是的，循环赛并不完美。但它比只有一个 P2.py 更好，并且您可以通过简单的随机化来解决这种偏差。
从多个 P2.py 扇入到一个 P3.py 还是有点复杂。此时，老式方法不再具有优势。P3.py 需要使用select库从多个命名管道中读取数据以交错读取。
我意识到我参加派对有点晚了,但我最近发现了GNU并行,并希望展示用它完成这个典型任务是多么容易.
cat input.csv | parallel ./sum.py --pipe > sums
这样的事情会做sum.py:
#!/usr/bin/python
from sys import argv
if __name__ == '__main__':
row = argv[-1]
values = (int(value) for value in row.split(','))
print row, ':', sum(values)
并行将为sum.py每一行input.csv(当然并行)运行,然后输出结果sums.显然比multiprocessing麻烦好