download_robots.sh
1#!/bin/bash
2if [ -f data/robots.txt ]; then
3 elapsed=$(($(date +%s) - $(date -r data/robots.txt +%s)))
4fi
5
6if [ ! -f data/robots.txt ] || [ $elapsed -gt 86400 ]; then
7 echo "downloading new robots.txt..."
8 wget -O data/robots.txt -q https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/refs/heads/master/dev-tools/_robots_input/robots-input.txt
9 echo "done"
10else
11 echo "robots.txt is new enough"
12fi