- commit
- e7f69d2
- parent
- f3e6112
- author
- Antonio Mika
- date
- 2025-03-08 10:49:35 -0500 EST
Add internal crawler matching
1 files changed,
+12,
-1
1@@ -21,6 +21,17 @@ import (
2 "github.com/x-way/crawlerdetect"
3 )
4
5+var internalCrawlers *crawlerdetect.CrawlerDetect
6+
7+func init() {
8+ internalCrawlers = crawlerdetect.New()
9+ internalCrawlers.SetCrawlers([]string{
10+ `^Azure Traffic Manager Endpoint Monitor$`,
11+ `^Blackbox Exporter\/`,
12+ `^Prometheus\/`,
13+ })
14+}
15+
16 func HmacString(secret, data string) string {
17 hmacer := hmac.New(sha256.New, []byte(secret))
18 hmacer.Write([]byte(data))
19@@ -30,7 +41,7 @@ func HmacString(secret, data string) string {
20
21 func trackableUserAgent(agent string) error {
22 // dont store requests from bots
23- if crawlerdetect.IsCrawler(agent) {
24+ if crawlerdetect.IsCrawler(agent) || internalCrawlers.IsCrawler(agent) {
25 return fmt.Errorf(
26 "request is likely from a bot (User-Agent: %s)",
27 CleanUserAgent(agent),