如何结合Scrapy输出

Question

我对 Scrapy 很陌生，这是迄今为止我最复杂的蜘蛛。

import scrapy
from scrapy.selector import HtmlXPathSelector


class CocabotSpider(scrapy.Spider):
    name = 'cocabot'
    start_urls = ['https://www.tallahasseearts.org/event/?keyword&start_date&end_date&date_format=m-d-Y&term=400&event_location&save_lst_list&view']
    custom_settings = {
        'FEED_URI' : 'output/cocaoutput.json'
    }

    def parse(self, response):
        # follow links to concert pages
        for href in response.css("div.search-img a::attr(href)"):
            yield response.follow(href, self.parse_concert)

        # follow links to venue pages
        for href in response.css("span.venue-event a::attr(href)"):
            yield response.follow(href, self.parse_venue)

        # follow links to pagination pages
        for href in response.css("li a.next.page-numbers::attr(href)"):
            yield response.follow(href, self.parse)

    def parse_concert(self, response):
        def extract_with_css(query):
            return response.css(query).extract_first()

        yield {
            'headliner' : extract_with_css("h1.p-ttl::text"),
            'venue' : extract_with_css("div.locatn div.a-block-ct div b::text"),
            'venue_address' : extract_with_css("div.locatn div.a-block-ct div p::text"),
            'venue_coca_url' : extract_with_css("span.venue-event a::attr(href)"),
            'event_url' : HtmlXPathSelector(response).select(
                "//div[@class='a-block-ct']/p/a[contains(text(), 'Official Website')]/@href")\
                .extract_first(),
            'event_coca_url' : response.request.url,
            'date_time' : extract_with_css("ul.ind-time li::text"),
            'price' : extract_with_css("div.a-block-ct div.apl-internal-content p::text"),
        }

    def parse_venue(self, response):
        yield {
            'venue_website' : HtmlXPathSelector(response).select(
                    "//div[@class='art-social-item']/a[contains(text(), 'Website')]/@href")\
                    .extract_first(),
        }

这是获取我想要的所有数据，但问题是venue_website数据在它自己的字典中。例子：

{"date_time": "Jun 18, 2018 at 08:00 am - 05:00 pm  (Mon)", "event_url": "http://www.music.fsu.edu/Quicklinks/Summer-Music-Camps/EXPLORE-OUR-14-CAMPS/Jazz-Ensemble-Camp-for-Middle-School", "venue_coca_url": null, "venue_address": "122 N. Copeland St., Tallahassee, FL 32304", "price": "Registration for camp is now open. You can register online or by mailing in a registration form. Day Camper Price: $311.00 (Includes tuition only. No housing or meals.) Night Camper Price: $501.00 \u2013 (Includes tuition and housing with three meals per day). A $100.00 non-refundable deposit is due at registration. Balance of camp fees are due by June 4.", "venue": "FSU College of Music", "headliner": "Jazz Ensemble Camp for Middle School", "event_coca_url": "https://www.tallahasseearts.org/event/jazz-ensemble-camp-for-middle-school-3/"},
{"venue_website": "http://www.makinglightproductions.org/"},
{"venue_website": "http://www.mfbooks.us/"},
{"venue_website": null},

我如何设法将venue_website 数据放入我的主要parse_concert 字典中？我尝试过在 parse_concert 函数中使用 follow 语句，并让 parse_venue 数据返回而不是 Yield，但我只是没有将它们正确地组合在一起。

Answer 1

scrapy中生成需要多页的item有两种方式：

1. 请求链接。

由于您需要多个请求来生成一项，因此您需要将它们按顺序链接起来并随身携带数据：

def parse_concert(self, response):
    concert = {'name': 'red hot chilly hotdogs'}
    venue_url = 'http://someplace.com'
    yield Request(venue_url, meta={'item': concert})

def parse_venue(self, response):
    item = response.meta['item']
    item['venue_name'] = 'someplace'
    yield item 
    # {'name': 'red hot chilly hotdogs', 'venue_name': 'someplace'}

2. 后处理合并。

另一种解决方案是异步生成两种类型的项目，然后通过共享 id 将它们组合起来：

def parse_concert(self, response):
    concert = {'name': 'red hot chilly hotdogs', 'id': 1}   
    yield concert
    yield Request(venue_url)

def parse_venue(self, response):
    item = {'venue_name': 'someplace', 'id': 1}
    yield item 

然后与替代脚本结合：

import json
with open('output.json') as f:
    data = json.loads(f.read())

combined = {}
for item in data:
    if item['id'] in combined:
        combined[item['id']].update(item)
    else:
        combined[item['id']] = item

with open('output_combined.json', 'w') as f:
        f.write(json.dumps(combined.values()))