module Squirm::Spider

Defined in:

squirm/spider.cr

Constant Summary

Log = ::Log.for(self)

Instance Method Summary

Instance Method Detail

abstract def base_url : String #

[View source]
abstract def cache : Caches::Base #

[View source]
abstract def concurrent_requests_per_domain : Int32 #

[View source]
abstract def fetcher : Fetchers::Base #

[View source]
abstract def id : String #

[View source]
abstract def parse_item(request : Request, response : Response) : ParsedItem #

[View source]
abstract def parser : Parser #

[View source]
abstract def request_filters : Array(RequestFilters::Base) #

[View source]
abstract def request_timeout : Time::Span #

[View source]
abstract def response_filters : Array(ResponseFilters::Base) #

[View source]
abstract def start_requests : Array(Request) #

[View source]
abstract def start_urls : Array(String) #

[View source]