The Web Scraping Ecosystem for Professional Developers

1from scrapy import Field, Item, Spider
2
3class MyItem(Item):
4    price = Field()
5
6
7class Scraper(Spider):
8
9   name = "scraper"
10   start_urls = ["https://the-best-store.com/"]
11
12   def parse(self, response, **kwargs):
13       item = MyItem()
14       item["price"] = response.css(".price_color::text").get()
15       return item

1def start_requests(self):
2        yield scrapy.Request(
3            url="https://quotes.toscrape.com/",
4            meta={
5              "zyte_api": {
6                "geolocation": "CA",
7                "browserHtml": True
8              }
9            }
10        )

1from scrapy import Request, Spider
2
3
4class MySpider(Spider):
5    my_spider = "toscrape_com"
6
7    def start_requests(self):
8        yield Request(
9            "https://books.toscrape.com/",
10            meta={
11                "zyte_api_automap": {
12                    "httpResponseBody": True,  
13                    "productList": True,  
14                },
15            },
16        )
17
18    def parse(self, response):
19        http_response_body: bytes = response.body
20        productList = response.raw_api_response["productList"]