golang源码分析：爬虫colly（part II）

type CollectorOption func(*Collector)

type Collector struct {
  // UserAgent is the User-Agent string used by HTTP requests
  UserAgent string
  // MaxDepth limits the recursion depth of visited URLs.
  // Set it to 0 for infinite recursion (default).
  MaxDepth int
  // AllowedDomains is a domain whitelist.
  // Leave it blank to allow any domains to be visited
  AllowedDomains []string
  // DisallowedDomains is a domain blacklist.
  DisallowedDomains []string
  // DisallowedURLFilters is a list of regular expressions which restricts
  // visiting URLs. If any of the rules matches to a URL the
  // request will be stopped. DisallowedURLFilters will
  // be evaluated before URLFilters
  // Leave it blank to allow any URLs to be visited
  DisallowedURLFilters []*regexp.Regexp
  // URLFilters is a list of regular expressions which restricts
  // visiting URLs. If any of the rules matches to a URL the
  // request won't be stopped. DisallowedURLFilters will
  // be evaluated before URLFilters
  // Leave it blank to allow any URLs to be visited
  URLFilters []*regexp.Regexp
  // AllowURLRevisit allows multiple downloads of the same URL
  AllowURLRevisit bool
  // MaxBodySize is the limit of the retrieved response body in bytes.
  // 0 means unlimited.
  // The default value for MaxBodySize is 10MB (10 * 1024 * 1024 bytes).
  MaxBodySize int
  // CacheDir specifies a location where GET requests are cached as files.
  // When it's not defined, caching is disabled.
  CacheDir string
  // IgnoreRobotsTxt allows the Collector to ignore any restrictions set by
  // the target host's robots.txt file.  See http://www.robotstxt.org/ for more
  // information.
  IgnoreRobotsTxt bool
  // Async turns on asynchronous network communication. Use Collector.Wait() to
  // be sure all requests have been finished.
  Async bool
  // ParseHTTPErrorResponse allows parsing HTTP responses with non 2xx status codes.
  // By default, Colly parses only successful HTTP responses. Set ParseHTTPErrorResponse
  // to true to enable it.
  ParseHTTPErrorResponse bool
  // ID is the unique identifier of a collector
  ID uint32
  // DetectCharset can enable character encoding detection for non-utf8 response bodies
  // without explicit charset declaration. This feature uses https://github.com/saintfish/chardet
  DetectCharset bool
  // RedirectHandler allows control on how a redirect will be managed
  // use c.SetRedirectHandler to set this value
  redirectHandler func(req *http.Request, via []*http.Request) error
  // CheckHead performs a HEAD request before every GET to pre-validate the response
  CheckHead bool
  // TraceHTTP enables capturing and reporting request performance for crawler tuning.
  // When set to true, the Response.Trace will be filled in with an HTTPTrace object.
  TraceHTTP bool
  // Context is the context that will be used for HTTP requests. You can set this
  // to support clean cancellation of scraping.
  Context context.Context




  store                    storage.Storage
  debugger                 debug.Debugger
  robotsMap                map[string]*robotstxt.RobotsData
  htmlCallbacks            []*htmlCallbackContainer
  xmlCallbacks             []*xmlCallbackContainer
  requestCallbacks         []RequestCallback
  responseCallbacks        []ResponseCallback
  responseHeadersCallbacks []ResponseHeadersCallback
  errorCallbacks           []ErrorCallback
  scrapedCallbacks         []ScrapedCallback
  requestCount             uint32
  responseCount            uint32
  backend                  *httpBackend
  wg                       *sync.WaitGroup
  lock                     *sync.RWMutex
}

            func (c *Collector) Init() 
func (c *Collector) Appengine(ctx context.Context)
func (c *Collector) Visit(URL string) error 
func (c *Collector) HasVisited(URL string) (bool, error)
func (c *Collector) HasPosted(URL string, requestData map[string]string) (bool, error)
func (c *Collector) Head(URL string) error
func (c *Collector) Request(method, URL string, requestData io.Reader, ctx *Context, hdr http.Header) error
func (c *Collector) UnmarshalRequest(r []byte) (*Request, error)
func (c *Collector) scrape(u, method string, depth int, requestData io.Reader, ctx *Context, hdr http.Header, checkRevisit bool) error
func setRequestBody(req *http.Request, body io.Reader) 
func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ctx *Context, hdr http.Header, req *http.Request) error
func (c *Collector) requestCheck(u string, parsedURL *url.URL, method string, requestData io.Reader, depth int, checkRevisit bool) error 
func (c *Collector) checkRobots(u *url.URL) error 
func (c *Collector) OnRequest(f RequestCallback){
      c.requestCallbacks = make([]RequestCallback, 0, 4)
}
func (c *Collector) OnResponseHeaders(f ResponseHeadersCallback){ 
      c.responseHeadersCallbacks = append(c.responseHeadersCallbacks, f)
}
func (c *Collector) handleOnRequest(r *Request) {
  if c.debugger != nil {
    c.debugger.Event(createEvent("request", r.ID, c.ID, map[string]string{
      "url": r.URL.String(),
    }))
  }
  for _, f := range c.requestCallbacks {
    f(r)
  }
}
func (c *Collector) handleOnHTML(resp *Response) error 

func (c *Collector) Visit(URL string) error {
      return c.scrape(URL, "GET", 1, nil, nil, nil, true)
}

 func (c *Collector) scrape(u, method string, depth int, requestData io.Reader, ctx *Context, hdr http.Header, checkRevisit bool) error{
      go c.fetch(u, method, depth, requestData, ctx, hdr, req)
      return c.fetch(u, method, depth, requestData, ctx, hdr, req)
}

func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ctx *Context, hdr http.Header, req *http.Request) error
      c.handleOnRequest(request)
      c.handleOnResponseHeaders(&Response{Ctx: ctx, Request: request, StatusCode: statusCode, Headers: &headers})
      err := c.handleOnError(response, err, request, ctx); err != nil 
      c.handleOnResponse(response)
      err = c.handleOnHTML(response)
      c.handleOnScraped(response)

  // RequestCallback is a type alias for OnRequest callback functions
type RequestCallback func(*Request)

// ResponseHeadersCallback is a type alias for OnResponseHeaders callback functions
type ResponseHeadersCallback func(*Response)

// ResponseCallback is a type alias for OnResponse callback functions
type ResponseCallback func(*Response)

// HTMLCallback is a type alias for OnHTML callback functions
type HTMLCallback func(*HTMLElement)

// XMLCallback is a type alias for OnXML callback functions
type XMLCallback func(*XMLElement)

// ErrorCallback is a type alias for OnError callback functions
type ErrorCallback func(*Response, error)

// ScrapedCallback is a type alias for OnScraped callback functions
type ScrapedCallback func(*Response)

// ProxyFunc is a type alias for proxy setter functions.
type ProxyFunc func(*http.Request) (*url.URL, error)

  var envMap = map[string]func(*Collector, string)
    ALLOWED_DOMAINS
    CACHE_DIR


  

  func NewCollector(options ...CollectorOption) *Collector
      c.Init()
      for _, f := range options {
         f(c)
      }
      c.parseSettingsFromEnv()

type Context struct {
  contextMap map[string]interface{}
  lock       *sync.RWMutex
}

type HTMLElement struct {
  // Name is the name of the tag
  Name       string
  Text       string
  attributes []html.Attribute
  // Request is the request object of the element's HTML document
  Request *Request
  // Response is the Response object of the element's HTML document
  Response *Response
  // DOM is the goquery parsed DOM object of the page. DOM is relative
  // to the current HTMLElement
  DOM *goquery.Selection
  // Index stores the position of the current element within all the elements matched by an OnHTML callback
  Index int
}  

func (h *HTMLElement) Attr(k string) string 
func (h *HTMLElement) ChildText(goquerySelector string) string
func (h *HTMLElement) ChildAttr(goquerySelector, attrName string) string

type httpBackend struct {
  LimitRules []*LimitRule
  Client     *http.Client
  lock       *sync.RWMutex
}

type LimitRule struct {
  // DomainRegexp is a regular expression to match against domains
  DomainRegexp string
  // DomainGlob is a glob pattern to match against domains
  DomainGlob string
  // Delay is the duration to wait before creating a new request to the matching domains
  Delay time.Duration
  // RandomDelay is the extra randomized duration to wait added to Delay before creating a new request
  RandomDelay time.Duration
  // Parallelism is the number of the maximum allowed concurrent requests of the matching domains
  Parallelism    int
  waitChan       chan bool
  compiledRegexp *regexp.Regexp
  compiledGlob   glob.Glob
}  

func (r *LimitRule) Match(domain string) bool
func (h *httpBackend) Do(request *http.Request, bodySize int, checkHeadersFunc checkHeadersFunc) (*Response, error) 
      res, err := h.Client.Do(request)

type HTTPTrace struct {
  start, connect    time.Time
  ConnectDuration   time.Duration
  FirstByteDuration time.Duration
}

type Request struct {
  // URL is the parsed URL of the HTTP request
  URL *url.URL
  // Headers contains the Request's HTTP headers
  Headers *http.Header
  // Ctx is a context between a Request and a Response
  Ctx *Context
  // Depth is the number of the parents of the request
  Depth int
  // Method is the HTTP method of the request
  Method string
  // Body is the request body which is used on POST/PUT requests
  Body io.Reader
  // ResponseCharacterencoding is the character encoding of the response body.
  // Leave it blank to allow automatic character encoding of the response body.
  // It is empty by default and it can be set in OnRequest callback.
  ResponseCharacterEncoding string
  // ID is the Unique identifier of the request
  ID        uint32
  collector *Collector
  abort     bool
  baseURL   *url.URL
  // ProxyURL is the proxy address that handles the request
  ProxyURL string
}

type Response struct {
  // StatusCode is the status code of the Response
  StatusCode int
  // Body is the content of the Response
  Body []byte
  // Ctx is a context between a Request and a Response
  Ctx *Context
  // Request is the Request object of the response
  Request *Request
  // Headers contains the Response's HTTP headers
  Headers *http.Header
  // Trace contains the HTTPTrace for the request. Will only be set by the
  // collector if Collector.TraceHTTP is set to true.
  Trace *HTTPTrace
}

func UnmarshalHTML(v interface{}, s *goquery.Selection, structMap map[string]string) error 

 type XMLElement struct {
  // Name is the name of the tag
  Name       string
  Text       string
  attributes interface{}
  // Request is the request object of the element's HTML document
  Request *Request
  // Response is the Response object of the element's HTML document
  Response *Response
  // DOM is the DOM object of the page. DOM is relative
  // to the current XMLElement and is either a html.Node or xmlquery.Node
  // based on how the XMLElement was created.
  DOM    interface{}
  isHTML bool
}  

func (h *XMLElement) ChildText(xpathQuery string) string