From d1662f520ed9fc2dfd40dfd873d4e0bf39a84f27 Mon Sep 17 00:00:00 2001
From: Earl Warren <contact@earl-warren.org>
Date: Sat, 13 Apr 2024 15:15:30 +0200
Subject: [PATCH] developer: infrastructure: blacklist bots that do not obey
 robots.txt

They get /forgejo/forgejo/archive/100448a007707c049a769cb71cd170fcb5b64a25.bundle

despite https://code.forgejo.org/robots.txt having

Disallow: /*/*/archive/
Disallow: /*/*/archive/*.bundle
---
 docs/developer/infrastructure.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/developer/infrastructure.md b/docs/developer/infrastructure.md
index 53fc4db3..6221d571 100644
--- a/docs/developer/infrastructure.md
+++ b/docs/developer/infrastructure.md
@@ -271,6 +271,8 @@ server {
     server_name code.forgejo.org;
 
     location / {
+        deny 47.76.209.138; # crawler that does not obey robots.txt
+        deny 47.76.99.127; # crawler that does not obey robots.txt
         proxy_pass http://10.6.83.195:8080;
         client_max_body_size 2G;
 	#