add OnError handle

jaeles-project · Feb 7, 2020 · ac22388 · ac22388
1 parent d981d45
commit ac22388
Showing 1 changed file with 10 additions and 1 deletion.
diff --git a/core/crawler.go b/core/crawler.go
@@ -279,13 +279,22 @@ func (crawler *Crawler) Start() {
 		u := response.Request.URL.String()
 		if crawler.domainRe.MatchString(u) {
 			outputFormat := fmt.Sprintf("[url] - [code-%d] - %s", response.StatusCode, u)
-			//Logger.Info(outputFormat + "\n")
 			fmt.Println(outputFormat)
 			if crawler.Output != nil {
 				crawler.Output.WriteToFile(outputFormat)
 			}
 		}
 	})
+
+	crawler.C.OnError(func(response *colly.Response, err error) {
+		u := response.Request.URL.String()
+		outputFormat := fmt.Sprintf("[url] - [code-%d] - %s", response.StatusCode, u)
+		fmt.Println(outputFormat)
+		if crawler.Output != nil {
+			crawler.Output.WriteToFile(outputFormat)
+		}
+	})
+
 	_ = crawler.C.Visit(crawler.site)
 }