#!/usr/bin/perl
use strict;
use warnings;
use Tie::File;
use Data::Dumper;
use Benchmark;
my $t0 = Benchmark->new;
# all files in the current folder with $ext will be input.
# Default $ext is "pileup"
# if entered, second user entered input will be set to $ext
my $ext = "pileup";
if(exists $ARGV[1]) {
$ext = $ARGV[1];
}
# open current directory & store filenames with $ext into @pileupfiles
opendir (DIR, ".");
my @pileupfiles = grep {-f && /\.$ext$/} readdir DIR;
my $dnasegment;
my $pos;
my $total;
my $g_total;
my @index; #hold current index for each tied file
my @totalfiles; #hold total files in each sub-index
# $filenum is iterator to cycle through all pileup files whose names are stored in pileupfiles
my $filenum = 0;
# @tied is an array holding all arrays of tied files
my @tied;
# array of the current line number for each @file,
my @linenum;
# tie each file to an array that is an element of the @tied array
while($filenum < scalar @pileupfiles) {
my @file;
tie @file, 'Tie::File', $pileupfiles[$filenum], recsep => "\n" or die;
push(@tied, [@file]);
# set each line's value of $linenum to 0
push(@linenum, 0);
$filenum++;
}
# open user list of dnasegments
open(LIST, $ARGV[0]);
# open file for output
open(OUT, ">>tempfile.tab");
while(<LIST>) {
$dnasegment = $_;
chomp $dnasegment;
my $exit = 0;
$pos = 1;
my %flag;
while(scalar(keys %flag) < scalar @tied) {
$total = 0;
$filenum = 0;
while($filenum < scalar @tied) {
if(exists $tied[$filenum][$linenum[$filenum]]) {
my @line = split(/\t/, $tied[$filenum][$linenum[$filenum]]);
#print $line[0], "\t", $line[1], "\t", $line[3], "\n\n";
if($line[0] eq $dnasegment) {
if($line[1] == $pos) {
$total += $line[3];
$linenum[$filenum]++;
$g_total += $line[3];
print OUT "$dnasegment\t$filenum\t$pos\t$line[3]\n";
}
} else {
$flag{$filenum} = 1;
}
} else {
#print $flag, "\n";
$flag{$filenum} = 1;
}
$filenum++;
}
if($total > 0) {
print OUT "$dnasegment\t$total\n";
}
$pos++;
}
}
close (LIST);
close(OUT);
my $t1 = Benchmark->new;
my $td = timediff($t1, $t0);
print timestr($td), "\n";
上面的代码在目录中获取具有默认或用户输入文件扩展名的所有文件,并计算特定条目(输入文件的第1列)中的位置(输入文件的第2列)的总出现次数(输入文件的第4列)第1列匹配命令行中提供的文件中包含的名称.程序使用的文件布局是:文件1:
Gm02 11896804 G 2 ., \'
Gm02 11896805 G 7 ......, U`
Gm02 11896806 G 3 .,. Sa
Gm02 11896807 T 2 ., U\
Gm02 11896808 T 2 ., ZZ
Gm02 11896809 T 2 ., ZZ
Gm02 11896810 T 2 ., B\
Gm02 11896811 G 3 .,^!, B]E
Gm02 11896812 A 3 T,, BaR
Gm02 11896822 G 3 .,, B`D
档案2:
Gm02 11896804 G 3 .,, \'
Gm02 11896805 G 7 ......, U`
Gm02 11896806 G 3 .,. Sa
Gm02 11896807 T 2 ., U\
Gm02 11896808 T 2 ., ZZ
Gm02 11896809 T 2 ., ZZ
Gm02 11896810 T 2 ., B\
Gm02 11896811 G 3 .,^!, B]E
Gm02 11896812 A 3 T,, BaR
Gm02 11896813 G 3 .,, B`D
档案3:
Gm02 11896804 G 3 .,, \'
Gm02 11896805 G 7 ......, U`
Gm02 11896806 G 3 .,. Sa
Gm02 11896807 T 2 ., U\
Gm02 11896808 T 2 ., ZZ
Gm02 11896809 T 2 ., ZZ
Gm02 11896810 T 2 ., B\
Gm02 11896811 G 3 .,^!, B]E
Gm02 11896812 A 3 T,, BaR
Gm02 11896833 G 3 .,, B`D
在这种情况下,传递给程序的唯一命令行参数将是一个文本文件,其中包含"Gm02"作为其内容.
哈希用于跟踪已经处理的位置.在上面的示例文件中,在遇到位置11896804的第一个值之前,将检查所有三个文件从位置1到11896803进行计数.这是为了确保在递增位置之前检查所有位置并在所有文件中求和.
我的问题与表现有关.我决定使用Tie :: File,因为我理解这会提高性能,因为所有文件都不会被读入内存.程序要处理的实际数据是数十万行,乘以数十个文件.此时,单独运行示例file1和所有3个示例文件所花费的时间分别为42个wallclock secs(41.96 usr + 0.00 sys = 41.96 CPU)和110个wallclock secs(109.76 usr + 0.00 sys = 109.76 CPU).任何关于为什么这个程序运行如此缓慢的信息或关于如何加快它的建议将非常感激.
编辑美国东部时间晚上10:17:该计划的输出如下:
Gm02 0 11896804 2
Gm02 1 11896804 3
Gm02 2 11896804 3
Gm02 8
Gm02 0 11896805 7
Gm02 1 11896805 7
Gm02 2 11896805 7
Gm02 21
Gm02 0 11896806 3
Gm02 1 11896806 3
Gm02 2 11896806 3
Gm02 9
Gm02 0 11896807 2
Gm02 1 11896807 2
Gm02 2 11896807 2
Gm02 6
Gm02 0 11896808 2
Gm02 1 11896808 2
Gm02 2 11896808 2
Gm02 6
Gm02 0 11896809 2
Gm02 1 11896809 2
Gm02 2 11896809 2
Gm02 6
Gm02 0 11896810 2
Gm02 1 11896810 2
Gm02 2 11896810 2
Gm02 6
Gm02 0 11896811 3
Gm02 1 11896811 3
Gm02 2 11896811 3
Gm02 9
Gm02 0 11896812 3
Gm02 1 11896812 3
Gm02 2 11896812 3
Gm02 9
Gm02 1 11896813 3
Gm02 3
Gm02 0 11896822 3
Gm02 3
Gm02 2 11896833 3
Gm02 3
Gm02 0 11896804 2
Gm02 1 11896804 3
Gm02 5
Gm02 0 11896805 7
Gm02 1 11896805 7
Gm02 14
Gm02 0 11896806 3
Gm02 1 11896806 3
Gm02 6
Gm02 0 11896807 2
Gm02 1 11896807 2
Gm02 4
Gm02 0 11896808 2
Gm02 1 11896808 2
Gm02 4
Gm02 0 11896809 2
Gm02 1 11896809 2
Gm02 4
Gm02 0 11896810 2
Gm02 1 11896810 2
Gm02 4
Gm02 0 11896811 3
Gm02 1 11896811 3
Gm02 6
Gm02 0 11896812 3
Gm02 1 11896812 3
Gm02 6
Gm02 1 11896813 3
Gm02 3
Gm02 0 11896822 3
Gm02 3
Gm02 0 11896804 2
Gm02 2
Gm02 0 11896805 7
Gm02 7
Gm02 0 11896806 3
Gm02 3
Gm02 0 11896807 2
Gm02 2
Gm02 0 11896808 2
Gm02 2
Gm02 0 11896809 2
Gm02 2
Gm02 0 11896810 2
Gm02 2
Gm02 0 11896811 3
Gm02 3
Gm02 0 11896812 3
Gm02 3
Gm02 0 11896822 3
Gm02 3
我会说"因为你正在使用Tie :: File",除了你不在以下代码行之外:
my @file;
tie @file, 'Tie::File', $pileupfiles[$filenum], recsep => "\n" or die;
push(@tied, [@file]);
你也可以把它写成
open(my $fh, '<', $pileupfiles[$filenum]) or die $!;
push(@tied, [ <$fh> ]);
也许你的意思
tie my @file, 'Tie::File', $pileupfiles[$filenum], recsep => "\n" or die;
push(@tied, \@file);
然后我们回到原来的答案.在某些情况下,Tie :: File可能会减少开发时间,但它不会成为迄今为止最快的解决方案,它可能会使用更多需要的内存.
顺便说一下,exist对数组元素没有意义.
if (exists $tied[$filenum][$linenum[$filenum]])
这是一种糟糕的做法
if (defined $tied[$filenum][$linenum[$filenum]])
要么
if ($linenum[$filenum] < @{ $tied[$filenum] })
| 归档时间： |
|
| 查看次数： |
281 次 |
| 最近记录： |