Some bots to put in robots.txt
User-agent: AdsBot-Google
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: Mediapartners-Google
Disallow: /
User-agent: Omgilibot
Disallow: /
User-Agent: panscient.com
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: Turnitin
Disallow: /
Dad /
npub123…ldvly
2024-03-03 21:08:11
Author Public Key
npub123pcea6k3yvz9r49hqv6sqmvdtvm28wxf3vwa6zeueqmzfcs3kxspldvlyPublished at
2024-03-03 21:08:11Event JSON
{
"id": "b55a28f2323e034ba1173c3eba10a98568624344606f485cb3e8c7ecb6c77545",
"pubkey": "54438cf7568918228ea5b819a8036c6ad9b51dc64c58eee859e641b127108d8d",
"created_at": 1709500091,
"kind": 1,
"tags": [
[
"proxy",
"https://mastodon.social/users/GeekAndDad/statuses/112033798009658738",
"activitypub"
]
],
"content": "Some bots to put in robots.txt\n\nUser-agent: AdsBot-Google\nDisallow: /\nUser-agent: Amazonbot\nDisallow: /\nUser-agent: Bytespider\nDisallow: /\nUser-agent: ClaudeBot\nDisallow: /\nUser-agent: Google-Extended\nDisallow: /\nUser-agent: GPTBot\nDisallow: /\nUser-agent: Mediapartners-Google\nDisallow: /\nUser-agent: Omgilibot\nDisallow: /\nUser-Agent: panscient.com\nDisallow: /\nUser-agent: PetalBot\nDisallow: /\nUser-agent: Turnitin\nDisallow: /",
"sig": "8d241614ca048f49fb2fbadf4962f9d0c78178501753133d0506c4028320067b6df8073a6d66a84119d646d5c5bbaf623379ea2eb12821a8a1bb38bea90c1e18"
}