如何在python请求库中实现重试机制？

Question

我想为python请求库添加一个重试机制,因此使用它的脚本将重试非致命错误.

此刻我确实认为有些错误是可以恢复的:

• HTTP返回码502,503,504
• 主机未找到(现在不太重要)
• 请求超时

在第一阶段,我确实想每分钟重试指定的5xx请求.

我希望能够透明地添加此功能,而无需为使用python-requests的这些脚本或库中的每个HTTP调用手动实现恢复.

Answer 1

这段代码将使来自同一会话的所有HTTP请求重试总共5次,在重试之间休眠,增加后退0s,2s,4s,8s,16s(第一次重试立即完成).它将重试基本连接问题(包括DNS查找失败)和HTTP状态代码502,503和504.

import logging
import requests

from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry

logging.basicConfig(level=logging.DEBUG)

s = requests.Session()
retries = Retry(total=5, backoff_factor=1, status_forcelist=[ 502, 503, 504 ])
s.mount('http://', HTTPAdapter(max_retries=retries))

s.get("http://httpstat.us/503")

有关详细信息,请参阅重试类.

Answer 2

使用重试包的可能解决方案

from retrying import retry
import requests


def retry_if_connection_error(exception):
    """ Specify an exception you need. or just True"""
    #return True
    return isinstance(exception, ConnectionError)

# if exception retry with 2 second wait  
@retry(retry_on_exception=retry_if_connection_error, wait_fixed=2000)
def safe_request(url, **kwargs):
    return requests.get(url, **kwargs)

response = safe_request('test.com')

Answer 3

这是我用来重试urllib2的请愿的代码片段。也许您可以将其用于您的目的：

retries = 1
success = False
while not success:
    try:
        response = urllib2.urlopen(request)
        success = True
    except Exception as e:
        wait = retries * 30;
        print 'Error! Waiting %s secs and re-trying...' % wait
        sys.stdout.flush()
        time.sleep(wait)
        retries += 1

等待时间逐渐增加，以避免被服务器禁止。

Answer 4

from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry


MAX_RETRY = 2
MAX_RETRY_FOR_SESSION = 2
BACK_OFF_FACTOR = 0.3
TIME_BETWEEN_RETRIES = 1000
ERROR_CODES = (500, 502, 504)


def requests_retry_session(retries=MAX_RETRY_FOR_SESSION,
    back_off_factor=BACK_OFF_FACTOR,
    status_force_list=ERROR_CODES, 
    session=None):
       session = session  
       retry = Retry(total=retries, read=retries, connect=retries,
                     backoff_factor=back_off_factor,
                     status_forcelist=status_force_list,
                     method_whitelist=frozenset(['GET', 'POST']))
       adapter = HTTPAdapter(max_retries=retry)
       session.mount('http://', adapter)
       session.mount('https://', adapter)
       return session



class ConfigService:

   def __init__(self):
      self.session = requests_retry_session(session=requests.Session())

   def call_to_api():
      config_url = 'http://localhost:8080/predict/'
      headers = {
        "Content-Type": "application/json",
        "x-api-key": self.x_api_key
      } 
      response = self.session.get(config_url, headers=headers)
      return response

Answer 5

通过扩展课程，我能够获得所需的可靠性水平requests.Session。

这是代码https://bitbucket.org/bspeakmon/jira-python/src/a7fca855394402f58507ca4056de87ccdbd6a213/jira/resilientsession.py?at=master

编辑该代码是：

from requests import Session
from requests.exceptions import ConnectionError
import logging
import time


class ResilientSession(Session):

    """
    This class is supposed to retry requests that do return temporary errors.

    At this moment it supports: 502, 503, 504
    """

    def __recoverable(self, error, url, request, counter=1):
        if hasattr(error,'status_code'):
            if error.status_code in [502, 503, 504]:
                error = "HTTP %s" % error.status_code
            else:
                return False
        DELAY = 10 * counter
        logging.warn("Got recoverable error [%s] from %s %s, retry #%s in %ss" % (error, request, url, counter, DELAY))
        time.sleep(DELAY)
        return True


    def get(self, url, **kwargs):
        counter = 0
        while True:
            counter += 1
            try:
                r = super(ResilientSession, self).get(url, **kwargs)
            except ConnectionError as e:
                r = e.message
            if self.__recoverable(r, url, 'GET', counter):
                continue
            return r

    def post(self, url, **kwargs):
        counter = 0
        while True:
            counter += 1
            try:
                r = super(ResilientSession, self).post(url, **kwargs)
            except ConnectionError as e:
                r = e.message
            if self.__recoverable(r, url, 'POST', counter):
                continue
            return r

    def delete(self, url, **kwargs):
        counter = 0
        while True:
            counter += 1
            try:
                r = super(ResilientSession, self).delete(url, **kwargs)
            except ConnectionError as e:
                r = e.message
            if self.__recoverable(r, url, 'DELETE', counter):
                continue
            return r

    def put(self, url, **kwargs):
        counter = 0
        while True:
            counter += 1
            try:
                r = super(ResilientSession, self).put(url, **kwargs)
            except ConnectionError as e:
                r = e.message

            if self.__recoverable(r, url, 'PUT', counter):
                continue
            return r

    def head(self, url, **kwargs):
        counter = 0
        while True:
            counter += 1
            try:
                r = super(ResilientSession, self).head(url, **kwargs)
            except ConnectionError as e:
                r = e.message
            if self.__recoverable(r, url, 'HEAD', counter):
                continue
            return r

    def patch(self, url, **kwargs):
        counter = 0
        while True:
            counter += 1
            try:
                r = super(ResilientSession, self).patch(url, **kwargs)
            except ConnectionError as e:
                r = e.message

            if self.__recoverable(r, url, 'PATCH', counter):
                continue
            return r

    def options(self, url, **kwargs):
        counter = 0
        while True:
            counter += 1
            try:
                r = super(ResilientSession, self).options(url, **kwargs)
            except ConnectionError as e:
                r = e.message

            if self.__recoverable(r, url, 'OPTIONS', counter):
                continue
            return r