scrapy spider code check

Question

so im' trying to scrape the website in the SgmlLinkExtractor parameters below website with scrapy, and this is what my spider looks like:

from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import HtmlXPathSelector
from desidime_sample.items import DesidimeItem
import string

class DesidimeSpider(CrawlSpider):
    name = "desidime"
    allowed_domains = ["desidime.com"]
    start_urls = ["http://www.desidime.com/forums/hot-deals-online"]
    rules = (
        Rule(SgmlLinkExtractor(allow=(), restrict_xpaths=('''//td[not(@*)]/div
        [not(@*)]/a[not(@class)]/@href''')), callback="parse_items", follow=True),
)
    def parse_items(self, response):
        hxs = HtmlXPathSelector(response)
        deals = hxs.select('''//div[@class='user-comment-text'][1]''')
        items = []
        for deals in deals:
            item = DesidimeItem()
            item["deal"]  = deals.select("//div[@class='user-comment-text'][1]/p/text()").extract()
            item["link"] = deals.select("//div[@class='user-comment-text'][1]/p[1]/a[1]/@href").extract()
            items.append(item)
        return items

It should be quite obvious what I'm trying to do, but for some reason when I tell the spider to crawl and export the text and links to the CVS file, I end up with:

link,deal http://wwww.facebook.com/desidime, http://wwww.facebook.com/desidime, (same thing for many more lines, then:) ",," , " same url" , (same thing for many more lines, then:) "link,deals"

So, can anyone tell me what the problem is? If you run each of my above xpaths as reponse.xpath("xpath").extract() after scrapy shell "//corresponingcrawlruleurl", you'll get the right results.

Answer 1

The problem is inside the parse_items callback. When you iterate over the deals, the deal context-specific locators have to be relative. In other words, start your XPath expressions inside the loop with a dot:

def parse_items(self, response):
    for deal in response.xpath("//div[@class='user-comment-text'][1]"):
        item = DesidimeItem()

        item["deal"]  = deal.xpath(".//p/text()").extract()
        item["link"] = deal.xpath(".//p[1]/a[1]/@href").extract()

        yield item

(note that I've also simplified the code).

Here is the complete spider, I'm executing (it does scrape the text and links, though I don't know what is your desired output):

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor


class DesidimeItem(scrapy.Item):
    deal = scrapy.Field()
    link = scrapy.Field()


class DesidimeSpider(CrawlSpider):
    name = "desidime"
    allowed_domains = ["desidime.com"]
    start_urls = ["http://www.desidime.com/forums/hot-deals-online"]

    rules = [
        Rule(LinkExtractor(restrict_xpaths="//td[not(@*)]/div[not(@*)]/a[not(@class)]"),
             callback="parse_items",
             follow=True),
    ]

    def parse_items(self, response):
        for deal in response.xpath("//div[@class='user-comment-text'][1]"):
            item = DesidimeItem()

            item["deal"] = deal.xpath(".//p/text()").extract()
            item["link"] = deal.xpath(".//p[1]/a[1]/@href").extract()

            yield item