如何在 Html Agility 包中获取重定向 URL

Question

我想从设置的 URL 中解析所有 URL。我找到了以下方法：

 public static List<string> ParseLinks(string urlToCrawl)
    {
        WebClient webClient = new WebClient();

        byte[] data = webClient.DownloadData(urlToCrawl);
        string download = Encoding.ASCII.GetString(data);

        HashSet<string> list = new HashSet<string>();

        var doc = new HtmlDocument();
        doc.LoadHtml(download);
        HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes("//a[@href]");

        foreach (var n in nodes)
        {
            string href = n.Attributes["href"].Value;
            list.Add(GetAbsoluteUrlString(urlToCrawl, href));
        }
        return list.ToList();
    }

    static string GetAbsoluteUrlString(string baseUrl, string url)
    {
        var uri = new Uri(url, UriKind.RelativeOrAbsolute);
        if (!uri.IsAbsoluteUri)
            uri = new Uri(new Uri(baseUrl), uri);
        return uri.ToString();
    }

一切都很好，但在某些网站中，链接通过他们的网站（他们正在重定向）。我有一个链接：https://www.houzz.com/trk/aHR0cHM6Ly9nb2xkbWFuYXJjaGl0ZWN0LmNvbS8/d76eaa05cc284c9f987d1d30948a6295/ue/MjgxNzk3OTg/84045ba5f6a5f8aa2c25d89b4e18 c788 。当我想使用我的方法提取链接时，ParseLinks 方法给了我错误的 URL，例如https://www.houzz.com/contact、https://www.houzz.com/site-map/ ...我的期望是https://goldmanarchitect.com/contact、https://goldmanarchitect.com/site-map/ ...因为当我们转到上面的链接时，它会重定向到https://goldmanarchitect.com/。那么，如何从当前URL获取重定向的页面呢？请给我一些解决方案来解决我的问题。

Answer 1

我对几个关键字进行了一些研究，并找到了解决我的问题的方法。以下方法解决了我的问题：

public static string GetFinalRedirect(string url)
{
    if(string.IsNullOrWhiteSpace(url))
        return url;

    int maxRedirCount = 8;  // prevent infinite loops
    string newUrl = url;
    do
    {
        HttpWebRequest req = null;
        HttpWebResponse resp = null;
        try
        {
            req = (HttpWebRequest) HttpWebRequest.Create(url);
            req.Method = "HEAD";
            req.AllowAutoRedirect = false;
            resp = (HttpWebResponse)req.GetResponse();
            switch (resp.StatusCode)
            {
                case HttpStatusCode.OK:
                    return newUrl;
                case HttpStatusCode.Redirect:
                case HttpStatusCode.MovedPermanently:
                case HttpStatusCode.RedirectKeepVerb:
                case HttpStatusCode.RedirectMethod:
                    newUrl = resp.Headers["Location"];
                    if (newUrl == null)
                        return url;

                    if (newUrl.IndexOf("://", System.StringComparison.Ordinal) == -1)
                    {
                        // Doesn't have a URL Schema, meaning it's a relative or absolute URL
                        Uri u = new Uri(new Uri(url), newUrl);
                        newUrl = u.ToString();
                    }
                    break;
                default:
                    return newUrl;
            }
            url = newUrl;
        }
        catch (WebException)
        {
            // Return the last known good URL
            return newUrl;
        }
        catch (Exception ex)
        {
            return null;
        }
        finally
        {
            if (resp != null)
                resp.Close();
        }
    } while (maxRedirCount-- > 0);

    return newUrl;
}