// Copyright 2018 Adam Tauber // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package debug import ( "encoding/json" "log" "net/http" "sync" "time" ) // WebDebugger is a web based debuging frontend for colly type WebDebugger struct { // Address is the address of the web server. It is 127.0.0.1:7676 by default. Address string initialized bool CurrentRequests map[uint32]requestInfo RequestLog []requestInfo sync.Mutex } type requestInfo struct { URL string Started time.Time Duration time.Duration ResponseStatus string ID uint32 CollectorID uint32 } // Init initializes the WebDebugger func (w *WebDebugger) Init() error { if w.initialized { return nil } defer func() { w.initialized = true }() if w.Address == "" { w.Address = "127.0.0.1:7676" } w.RequestLog = make([]requestInfo, 0) w.CurrentRequests = make(map[uint32]requestInfo) http.HandleFunc("/", w.indexHandler) http.HandleFunc("/status", w.statusHandler) log.Println("Starting debug webserver on", w.Address) go http.ListenAndServe(w.Address, nil) return nil } // Event updates the debugger's status func (w *WebDebugger) Event(e *Event) { w.Lock() defer w.Unlock() switch e.Type { case "request": w.CurrentRequests[e.RequestID] = requestInfo{ URL: e.Values["url"], Started: time.Now(), ID: e.RequestID, CollectorID: e.CollectorID, } case "response", "error": r := w.CurrentRequests[e.RequestID] r.Duration = time.Since(r.Started) r.ResponseStatus = e.Values["status"] w.RequestLog = append(w.RequestLog, r) delete(w.CurrentRequests, e.RequestID) } } func (w *WebDebugger) indexHandler(wr http.ResponseWriter, r *http.Request) { wr.Write([]byte(` Colly Debugger WebUI

Current Requests

Finished Requests

`)) } func (w *WebDebugger) statusHandler(wr http.ResponseWriter, r *http.Request) { w.Lock() jsonData, err := json.MarshalIndent(w, "", " ") w.Unlock() if err != nil { panic(err) } wr.Write(jsonData) }