小编Can*_*cil的帖子

我有这个基本的实用程序类并行获取(可能)缩短的URL并返回一个具有最终URL的字典.它使用此博客文章中描述的wait_any功能.

class UrlFetcher(object):

  @classmethod
  def fetch_urls(cls,url_list):
    rpcs = []
    for url in url_list:
      rpc = urlfetch.create_rpc(deadline=5.0)
      urlfetch.make_fetch_call(rpc, url,method = urlfetch.HEAD)
      rpcs.append(rpc)

    result = {}
    while len(rpcs) > 0:
      rpc = apiproxy_stub_map.UserRPC.wait_any(rpcs)
      rpcs.remove(rpc)
      request_url = rpc.request.url()
      try:
        final_url = rpc.get_result().final_url
      except AttributeError:
        final_url = request_url
      except DeadlineExceededError:
        logging.error('Handling DeadlineExceededError for url: %s' %request_url)
        final_url  = None
      except (DownloadError,InvalidURLError):
        final_url  = None        
      except UnicodeDecodeError: #Funky url with very evil characters
        final_url = unicode(rpc.get_result().final_url,'utf-8')

      result[request_url] = final_url

    logging.info('Returning results: %s' %result) …