robots.go
1package middleware
2
3import (
4 _ "embed"
5 "slices"
6 "strings"
7 "sync/atomic"
8
9 "github.com/bits-and-blooms/bloom"
10)
11
12type robots struct {
13 bloom *bloom.BloomFilter
14}
15
16var __robots_filter atomic.Pointer[robots]
17
18//go:generate /bin/bash ./data/download_robots.sh
19//go:embed data/robots.txt
20var robotsTxt string
21
22//go:embed data/whitelist.txt
23var whitelist string
24
25func init() {
26 __robots_filter.Store(&robots{newFilter()})
27}
28
29// IsRobot tests whether the user agent string is a known bot
30func IsRobot(userAgent string) bool {
31 if len(userAgent) == 0 {
32 return false
33 }
34 return __robots_filter.Load().bloom.Test([]byte(userAgent))
35}
36
37func newFilter() *bloom.BloomFilter {
38 ss := getList(robotsTxt)
39 wl := getList(whitelist)
40 b := bloom.NewWithEstimates(uint(len(ss)), 0.01)
41 for _, s := range ss {
42 if !slices.Contains(wl, s) {
43 b.Add([]byte(s))
44 }
45 }
46 return b
47}
48
49func getList(s string) []string {
50 ss := strings.Split(s, "\n")
51 out := make([]string, 0)
52 for _, s := range ss {
53 s = strings.Trim(s, " \r\n")
54 if len(s) > 0 {
55 out = append(out, s)
56 }
57 }
58 return out
59}
