From 092b0f57fb2d3aa656d1236e321280f346f97807 Mon Sep 17 00:00:00 2001
From: Wietse Wind <mail@wietse.com>
Date: Tue, 7 Nov 2023 00:57:54 +0100
Subject: [PATCH] Add robots txt & noindex

---
 bin/webserver.mjs | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/bin/webserver.mjs b/bin/webserver.mjs
index a877d47..6f37052 100644
--- a/bin/webserver.mjs
+++ b/bin/webserver.mjs
@@ -84,11 +84,19 @@ if (!wss) {
 
       wss = expressWs(app)
 
-      // app.use(function middlware (req, res, next) {
-      //   return next()
-      // })
+      app.use(function middlware (req, res, next) {
+        res.setHeader('X-Robots-Tag', 'noindex')
+        return next()
+      })
 
-      app.use('/', 
+      app.use('/robots.txt', (req, res, next) => {
+        res.setHeader('content-type', 'text/plain')
+        return res.send(
+          `User-agent: *\nDisallow: /\n`
+        )
+      })
+
+      app.use('/',
         (req, res, next) => {
           if (process.env?.TELEMETRY === 'YES' && !req.url.match(/health/)) {
             const telemetryData = {