blob: a403c38c744482a66eb13ae0b9e3441c6bd30e97 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
|
# Define networks to be used later below
networks:
# Networks will get included from snippets
challenges:
# Challenges will get included from snippets
conditions:
# Conditions will get replaced on rules AST when found as ($condition-name)
# Conditions will get included from snippets
is-static-asset:
- 'path == "/apple-touch-icon.png"'
- 'path == "/apple-touch-icon-precomposed.png"'
- 'path.matches("\\.(manifest|ttf|woff|woff2|jpg|jpeg|gif|png|webp|avif|svg|mp4|webm|css|js|mjs|wasm)$")'
is-suspicious-crawler:
- 'userAgent.contains("Presto/") || userAgent.contains("Trident/")'
# Old IE browsers
- 'userAgent.matches("MSIE ([2-9]|10|11)\\.")'
# Old Linux browsers
- 'userAgent.matches("Linux i[63]86") || userAgent.matches("FreeBSD i[63]86")'
# Old Windows browsers
- 'userAgent.matches("Windows (3|95|98|CE)") || userAgent.matches("Windows NT [1-5]\\.")'
# Old mobile browsers
- 'userAgent.matches("Android [1-5]\\.") || userAgent.matches("(iPad|iPhone) OS [1-9]_")'
# Old generic browsers
- 'userAgent.startsWith("Opera/")'
#- 'userAgent.matches("Gecko/(201[0-9]|200[0-9])")'
- 'userAgent.matches("^Mozilla/[1-4]")'
# Rules are checked sequentially in order, from top to bottom
rules:
- name: allow-well-known-resources
conditions:
- '($is-well-known-asset)'
action: pass
- name: allow-static-resources
conditions:
- '($is-static-asset)'
action: pass
- name: desired-crawlers
conditions:
- *is-bot-googlebot
- *is-bot-bingbot
- *is-bot-duckduckbot
- *is-bot-kagibot
- *is-bot-qwantbot
- *is-bot-yandexbot
action: pass
# Matches private networks and localhost.
# Uncomment this if you want to let your own tools this way
# - name: allow-private-networks
# conditions:
# # Allows localhost and private networks CIDR
# - *is-network-localhost
# - *is-network-private
# action: pass
- name: undesired-crawlers
conditions:
- '($is-headless-chromium)'
- 'userAgent.startsWith("Lightpanda/")'
- 'userAgent.startsWith("masscan/")'
# Typo'd opera botnet
- 'userAgent.matches("^Opera/[0-9.]+\\.\\(")'
# AI bullshit stuff, they do not respect robots.txt even while they read it
# TikTok Bytedance AI training
- 'userAgent.contains("Bytedance") || userAgent.contains("Bytespider") || userAgent.contains("TikTokSpider")'
# Meta AI training; The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly.
- 'userAgent.contains("meta-externalagent/") || userAgent.contains("meta-externalfetcher/") || userAgent.contains("FacebookBot")'
# Anthropic AI training and usage
- 'userAgent.contains("ClaudeBot") || userAgent.contains("Claude-User")|| userAgent.contains("Claude-SearchBot")'
# Common Crawl AI crawlers
- 'userAgent.contains("CCBot")'
# ChatGPT AI crawlers https://platform.openai.com/docs/bots
- 'userAgent.contains("GPTBot") || userAgent.contains("OAI-SearchBot") || userAgent.contains("ChatGPT-User")'
# Other AI crawlers
- 'userAgent.contains("Amazonbot") || userAgent.contains("Google-Extended") || userAgent.contains("PanguBot") || userAgent.contains("AI2Bot") || userAgent.contains("Diffbot") || userAgent.contains("cohere-training-data-crawler") || userAgent.contains("Applebot-Extended")'
# SEO / Ads and marketing
- 'userAgent.contains("BLEXBot")'
action: drop
- name: unknown-crawlers
conditions:
# No user agent set
- 'userAgent == ""'
action: deny
# check a sequence of challenges
- name: suspicious-crawlers
conditions: ['($is-suspicious-crawler)']
action: none
children:
- name: 0
action: check
settings:
challenges: [js-refresh]
- name: 1
action: check
settings:
challenges: [preload-link, resource-load]
- name: 2
action: check
settings:
challenges: [header-refresh]
- name: homesite
conditions:
- 'path == "/"'
- 'path == "/links/~ssh"'
- 'path == "/links/~pgp"'
action: pass
# check DNSBL and serve harder challenges
# todo: make this specific to score
- name: undesired-dnsbl
action: check
settings:
challenges: [dnsbl]
# if DNSBL fails, check additional challenges
fail: check
fail-settings:
challenges: [js-refresh]
- name: suspicious-fetchers
action: check
settings:
challenges: [js-refresh]
conditions:
- 'userAgent.contains("facebookexternalhit/") || userAgent.contains("facebookcatalog/")'
# Allow PUT/DELETE/PATCH/POST requests in general
- name: non-get-request
action: pass
conditions:
- '!(method == "HEAD" || method == "GET")'
# Enable fetching OpenGraph and other tags from backend on these paths
- name: enable-meta-tags
action: context
settings:
context-set:
# Map OpenGraph or similar <meta> tags back to the reply, even if denied/challenged
proxy-meta-tags: "true"
# Set additional response headers
#response-headers:
# X-Clacks-Overhead:
# - GNU Terry Pratchett
- name: plaintext-browser
action: challenge
settings:
challenges: [meta-refresh, cookie]
conditions:
- 'userAgent.startsWith("Lynx/")'
# Uncomment this rule out to challenge tool-like user agents
#- name: standard-tools
# action: challenge
# settings:
# challenges: [cookie]
# conditions:
# - '($is-generic-robot-ua)'
# - '($is-tool-ua)'
# - '!($is-generic-browser)'
- name: standard-browser
action: challenge
settings:
challenges: [preload-link, meta-refresh, resource-load, js-refresh, js-pow-sha256]
conditions:
- '($is-generic-browser)'
# If end of rules is reached, default is PASS
|
