##\
# This file is a part of the CIDRAM package.
# Homepage: https://cidram.github.io/
#
# CIDRAM COPYRIGHT 2016 and beyond by Caleb Mazalevskis (Maikuolan).
#
# License: GNU/GPLv2
# @see LICENSE.txt
#
# This file: Verification data for search engines and social media (last modified: 2019.06.30).
##/
Search Engine Verification:
# https://support.google.com/webmasters/answer/80553?hl=en
Googlebot:
Bypass Flag: "Flag-Bypass-Googlebot-Check"
User Agent: "googlebot"
Closure: "DNS-Reverse-Forward"
Valid Domains:
- ".googlebot.com"
- ".google.com"
# https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot
Bingbot:
Bypass Flag: "Flag-Bypass-Bingbot-Check"
User Agent Pattern: "~(?:msn|bing)bot|bingpreview~"
Closure: "DNS-Reverse-Forward"
Valid Domains:
- ".search.msn.com"
# http://www.ysearchblog.com/2007/06/05/yahoo-search-crawler-slurp-has-a-new-address-and-signature-card/
Y!Slurp:
Bypass Flag: "Flag-Bypass-Y!Slurp-Check"
User Agent: "slurp"
Closure: "DNS-Reverse-Forward"
Valid Domains:
- ".crawl.yahoo.net"
- ".yse.yahoo.net"
# https://help.baidu.com/question?prod_en=master&class=Baiduspider
Baidu spider:
Bypass Flag: "Flag-Bypass-Baidu-Check"
User Agent: "baidu"
Closure: "DNS-Reverse-Forward"
Valid Domains:
- ".baidu.com"
- ".baidu.jp"
# (No reference information currently available for Sogou web spider; Figured it out manually).
Sogou web spider:
Bypass Flag: "Flag-Bypass-Sogou-Check"
User Agent: "sogou"
Closure: "DNS-Reverse-Forward"
Valid Domains:
- ".sogou.com"
# (No reference information currently available for YoudaoBot; Figured it out manually).
YoudaoBot:
Bypass Flag: "Flag-Bypass-YoudaoBot-Check"
User Agent: "youdaobot"
Closure: "DNS-Reverse-Forward"
Valid Domains:
- ".crawler.youdao.com"
# https://discussions.apple.com/thread/7090135
Applebot:
Bypass Flag: "Flag-Bypass-Applebot-Check"
User Agent: "applebot"
Closure: "DNS-Reverse-Forward"
Valid Domains:
- ".applebot.apple.com"
# https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml
YandexBot:
Bypass Flag: "Flag-Bypass-Yandex-Check"
User Agent: "yandex"
Closure: "DNS-Reverse-Forward"
Valid Domains:
- ".yandex.com"
- ".yandex.net"
- ".yandex.ru"
# https://duckduckgo.com/duckduckbot
DuckDuckGo Bot:
Bypass Flag: "Flag-Bypass-DuckDuckGo-Check"
User Agent Pattern: "~duckduck(?:go-favicons-)?bot~"
Closure: "UA-IP-Match"
Can Modify Trackable: false
Valid Domains:
- "23.21.226.191"
- "54.208.102.37"
- "107.20.237.51"
- "107.21.1.8"
# https://napoveda.seznam.cz/en/full-text-search/seznambot-crawler/
SeznamBot:
Bypass Flag: "Flag-Bypass-SeznamBot-Check"
User Agent: "seznambot"
Closure: "DNS-Reverse-Forward"
Valid Domains:
- ".seznam.cz"
# https://www.distilnetworks.com/bot-directory/bot/teoma/
Teoma Bot:
Bypass Flag: "Flag-Bypass-Teoma-Check"
User Agent: "mozilla/2.0 (compatible; ask jeeves/teoma"
Closure: "UA-IP-Match"
Can Modify Trackable: false
Valid Domains:
- "63.123.238.8"
- "65.214.45.143"
- "65.214.45.148"
- "66.235.124.7"
- "66.235.124.73"
- "66.235.124.74"
- "66.235.124.101"
- "66.235.124.192"
- "66.235.124.193"
- "66.235.124.196"
- "202.143.148.61"
# https://www.mojeek.com/bot.html
MojeekBot:
Bypass Flag: "Flag-Bypass-Mojeek-Check"
User Agent: "mojeekbot"
Closure: "DNS-Reverse-Forward"
Valid Domains:
- ".mojeek.com"
Social Media Verification:
# https://help.pinterest.com/en/articles/about-pinterest-crawler-0
Pinterest:
User Agent: "pinterest"
Closure: "DNS-Reverse-Forward"
Valid Domains:
- ".pinterest.com"
# (No reference information currently available for Embedly; Figured it out manually).
Embedly:
User Agent: "embedly"
Closure: "DNS-Reverse-Forward"
Reverse Only: true
Can Modify Trackable: false
Valid Domains:
- "embed.ly"
# https://www.grapeshot.com/crawler/
GrapeshotCrawler:
User Agent: "grapeshot"
Closure: "DNS-Reverse-Forward"
Valid Domains:
- ".grapeshot.co.uk"
# (No reference information currently available for Twitterbot; Figured it out manually).
Twitterbot:
User Agent: "twitterbot"
Closure: "DNS-Reverse-Forward"
Valid Domains:
- ".twttr.com"
|
Download
