class Squirm::RequestFilters::Robots

Defined in:

squirm/request_filters/robots.cr

Constant Summary

Log = ::Log.for(self)

Constructors

.new(base_url : String, user_agent : String)

Instance Method Summary

#parser : ::Robots::Parser
#valid?(request : Request, spider : Spider) : Bool

Instance methods inherited from class `Squirm::RequestFilters::Base`

Constructor Detail

def self.new(base_url : String, user_agent : String) #

[View source]

Instance Method Detail

def parser : ::Robots::Parser #

[View source]

def valid?(request : Request, spider : Spider) : Bool #

[View source]