使用progressbar/ruby​​-progressbar gem时Zlib :: BufError

Question

我使用以下Ruby代码段下载8.9MB文件.

require 'open-uri'
require 'net/http'
require 'uri'

def http_download_no_progress_bar(uri, filename)
  uri.open(read_timeout: 500) do |file|
    open filename, 'w' do |io|
      file.each_line do |line|
        io.write line
      end
    end
  end
end

我想添加progressbar gem来可视化下载过程:

require 'open-uri'
require 'progressbar'
require 'net/http'
require 'uri'

def http_download_with_progressbar(uri, filename)
  progressbar = nil
  uri.open(
    read_timeout: 500,
    content_length_proc: lambda { |total|
    if total && 0 < total.to_i
      progressbar = ProgressBar.new("...", total)
      progressbar.file_transfer_mode
    end
    },
    progress_proc: lambda { |step|
       progressbar.set step if progressbar
    }
  ) do |file|
    open filename, 'w' do |io|
      file.each_line do |line|
        io.write line
      end
    end
  end
end

但是,现在它失败并出现以下错误:

/home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http/response.rb:357:in `finish': 
buffer error (Zlib::BufError)oooooo  |   8.0MB   8.6MB/s ETA:   0:00:00

    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http/response.rb:357:in `finish'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http/response.rb:262:in `ensure in inflater'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http/response.rb:262:in `inflater'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http/response.rb:274:in `read_body_0'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http/response.rb:201:in `read_body'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:328:in `block (2 levels) in open_http'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http.rb:1415:in `block (2 levels) in transport_request'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http/response.rb:162:in `reading_body'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http.rb:1414:in `block in transport_request'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http.rb:1405:in `catch'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http.rb:1405:in `transport_request'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http.rb:1378:in `request'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:319:in `block in open_http'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http.rb:853:in `start'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:313:in `open_http'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:724:in `buffer_open'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:210:in `block in open_loop'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:208:in `catch'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:208:in `open_loop'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:149:in `open_uri'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:704:in `open'

同时我也尝试了ruby-progressbar gem:

require 'open-uri'
require 'ruby-progressbar'
require 'net/http'
require 'uri'

def http_download_with_ruby_progressbar(uri, filename)
  progressbar = nil
  uri.open(
    read_timeout: 500,
    content_length_proc: lambda { |total|
      if total && 0 < total.to_i
        progressbar = ProgressBar.create(title: filename, total: total)
      end
      },
      progress_proc: lambda { |step|
        progressbar.progress = step if progressbar
      }
  ) do |file|
    open filename, 'w' do |io|
      file.each_line do |line|
        io.write line
      end
    end
  end
end

它失败并出现同样的错误.以下是该问题的相关问题.

Answer 1

问题是您尝试下载的文件,因为每个方法都适用于此文件:https://androidnetworktester.googlecode.com/files/1mb.txt.

问题是你的文件比它说的要大.content_length_proc表示它是8549968bytes(8.15MB)而它是101187668bytes(96.5MB)(在下载文件后用ls检查).现在我有一个不会崩溃的替代方案并为您提供进度条:

def http_download_with_words(uri, filename)
  bytes_total = nil
  uri.open(
           read_timeout: 500,
           :content_length_proc => lambda{|content_length|
             bytes_total = content_length},
           :progress_proc => lambda{|bytes_transferred|
             if bytes_total
               # Print progress
               print("\r#{bytes_transferred}/#{bytes_total}")
             else
               # We don’t know how much we get, so just print number
               # of transferred bytes
               print("\r#{bytes_transferred} (total size unknown)")
             end
             }
           ) do |file|
    open filename, 'w' do |io|
      file.each_line do |line|
        io.write line
      end
    end
  end
end

http_download_with_words(URI( 'http://data.wien.gv.at/daten/geo?service=WFS&request=GetFeature&version=1.1.0&typeName=ogdwien%3aBAUMOGD&srsName=EPSG:4326' ), 'temp.txt')

这是非常明显的,(见这里.)

现在我无法弄清楚的部分是进度条宝石究竟是如何干扰ZLib的.大多数东西似乎在procs中工作正常(例如让它们打印随机的东西)所以我假设这两个进度条在完成时做了一些奇怪的事情,不知何故混淆了转移.如果有人能弄明白为什么会这样,我会很感兴趣吗？