使用F#下载Windows Phone应用评论

Question

我正在自己学习F#(这很有趣,不适合工作/学校)而且我正在尝试编写一个简单的解析器来计算Windows Phone应用程序在多个市场中的评论数量.毫无疑问,我到目前为止的代码是丑陋的,但我正在努力改进它并遵循函数式编程范式.由于我来自C,C++,C#世界,所以很难.

• 来自C世界,我喜欢null值.我知道函数式编程/ F#不鼓励使用null,但我无法找到一种不使用它的方法.例如,在函数解析中有一个空检查.我怎么不这样做？

• 现在我的代码只计算第一页上的评论数量,但有可能一个应用程序有超过10条评论,因此可能有多个页面.我如何递归浏览所有页面(功能downloadReviews或parse).

• 我们怎样才能将此代码扩展为完全异步？

下面是我到目前为止的代码.除了上面的问题,我真的希望有人能帮助我,并告诉我如何改进我的代码的整体结构.

open System
open System.IO
open System.Xml
open System.Xml.Linq
open Printf

type DownloadPageResult = {
    Uri: System.Uri;
    ErrorOccured: bool;
    Source: string;
}

type ReviewData = {
    CurrentPageUri: System.Uri;
    NextPageUri: System.Uri;
    NumberOfReviews: int;
}

module ReviewUrl = 
    let getBaseUri path =
        new Uri(sprintf "http://cdn.marketplaceedgeservice.windowsphone.com/%s" path)

    let getUri country locale appId =
        getBaseUri(sprintf "/v8/ratings/product/%s/reviews?os=8.0.0.0&cc=%s&oc=&lang=%s&hw=520170499&dm=Test&chunksize=10" appId country locale)

let downloadPage (uri: System.Uri) =
    try
        use webClient = new System.Net.WebClient()
        printfn "%s" (uri.ToString())
        webClient.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
        webClient.Headers.Add("Accept-Encoding", "zip,deflate,sdch")
        webClient.Headers.Add("Accept-Language", "en-US,en;q=0.8,fr;q=0.6")
        webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1482.0 Safari/537.36")
        { Uri = uri; Source = webClient.DownloadString(uri); ErrorOccured = false }
    with error -> { Uri = uri; Source = String.Empty; ErrorOccured = true }

let downloadReview country locale appId =
    let uri = ReviewUrl.getUri country locale appId
    downloadPage uri

let parse(pageResult: DownloadPageResult) =
    if pageResult.ErrorOccured then { CurrentPageUri = pageResult.Uri; NextPageUri = null; NumberOfReviews = 0 }
    else 
        let reader = new StringReader(pageResult.Source)
        let doc = XDocument.Load(reader)
        let ns = XNamespace.Get("http://www.w3.org/2005/Atom")

        let nextUrl = query { for link in doc.Descendants(ns + "link") do
                              where (link.Attribute(XName.Get("rel")).Value = "next")
                              select link.Value
                              headOrDefault }

        if nextUrl = null then
            { CurrentPageUri = pageResult.Uri; NextPageUri = null; NumberOfReviews = doc.Descendants(ns + "entry") |> Seq.length } 
        else
            { CurrentPageUri = pageResult.Uri; NextPageUri = ReviewUrl.getBaseUri(nextUrl); NumberOfReviews = doc.Descendants(ns + "entry") |> Seq.length } 

let downloadReviews(locale: string) =
    let appId = "4e08377c-1240-4f80-9c35-0bacde2c66b6"
    let country = locale.Substring(3)
    let pageResult = downloadReview country locale appId
    let parseResult = parse pageResult
    parseResult


[<EntryPoint>]
let main argv = 
    let locales = [| "en-US"; "en-GB"; |]
    let results = locales |> Array.map downloadReviews

    printfn "%A" results
    0

Answer 1

我正在尝试更多这个问题并尝试使用XML类型提供程序和F#Data的其他功能.它不是完整的代码,但它应该足以给你一个想法(并显示类型提供者真的很好:-)):

首先,我需要一些参考:

#r "System.Xml.Linq.dll"
#r "FSharp.Data.dll"
open FSharp.Data
open FSharp.Net

接下来,我编写了以下代码来下载一个示例页面.

let data =
  Http.Request
   ( "http://cdn.marketplaceedgeservice.windowsphone.com//v8/ratings/product/4e08377c-1240-4f80-9c35-0bacde2c66b6/reviews",
     query=["os", "8.0.0.0"; "cc", "US"; "lang", "en-US"; "hw", "520170499"; "dm", "Test"; "chunksize", "10" ],
     headers=["User-Agent", "F#"])

我保存了样本D:\temp\appstore.xml,然后使用XML类型提供程序来获取一个很好的类型来解析页面:

type PageDocument = XmlProvider< @"D:\temp\appstore.xml" >

然后你可以像这样下载和解析页面(这显示了如何获得评论的数量和有关下一个链接的信息):

let parseAsync (locale:string) appId = async {
  let country = locale.Substring(3)

  // Make the request (asynchronously) using the parameters specified
  let! data =
    Http.AsyncRequest
     ( "http://cdn.marketplaceedgeservice.windowsphone.com//v8/ratings/product/" 
         + appId + "/reviews",
       query=[ "os", "8.0.0.0"; "cc", country; "lang", locale; 
               "hw", "520170499"; "dm", "Test"; "chunksize", "10" ],
       headers=["User-Agent", "F#"])

  // Parse the result using the type-provider generated type
  let page = PageDocument.Parse(data)

  // Now you can type 'page' followed by '.' and explore the results!
  // page.GetLinks() returns all links and page.GetEntries() returns
  // review entries. Each link also has 'Rel' and 'Href' properties:
  let nextLink = 
    page.GetLinks() 
    |> Seq.tryFind (fun link -> link.Rel = "next")
    |> Option.map (fun link -> link.Href)
  let reviewsCount = page.GetEntries().Length    
  return (reviewsCount, nextLink) }