tools/un-cloudflare.sh


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

#!/bin/bash
# Remove cloudflare instances from services-full.json

rm -f out.json
file="services-full.json"

while read -r line; do
    if [[ "$line" == "\"https://"* ]]; then
        domain=$(echo "$line" | sed -e "s/^\"https:\/\///" -e "s/\",//" -e "s/\"//")
        ns=$(dig ns "$domain" || true)
        if [[ "$ns" == *"cloudflare"* ]]; then
            echo "\"$domain\" using cloudflare, skipping..."
        elif [ ${#ns} -eq 0 ]; then
            echo "Unable to verify records for \"$domain\", skipping..."
        else
            echo "$line" >> out.json
        fi
    else
        echo "$line" >> out.json
    fi
done <$file

# Remove any trailing commas from new instance lists
sed -i '' -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json

cat out.json | jq --indent 2 . > services.json
rm -f out.json