From cf6810dfcdeae0ec1a5f25c266a8af68923a8fa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9D=93=99=F0=9D=93=BE=F0=9D=93=B5=F0=9D=93=B2?= =?UTF-8?q?=F0=9D=93=B8?= Date: Wed, 19 Jun 2024 02:47:46 +0100 Subject: [PATCH] Add nginx/bots.conf --- nginx/bots.conf | 664 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 664 insertions(+) create mode 100644 nginx/bots.conf diff --git a/nginx/bots.conf b/nginx/bots.conf new file mode 100644 index 0000000..976f92a --- /dev/null +++ b/nginx/bots.conf @@ -0,0 +1,664 @@ +map $http_user_agent $block_bot { + default 0; + "" 1; + "~*Bytespider" 1; + "~*AhrefsBot" 1; + "~*PetalBot" 1; + "~*DataForSeoBot" 1; + "~*SemrushBot" 1; + "~*SeekportBot" 1; + "~*ClaudeBot" 1; + "~*FeedBurner" 1; + "~*DotBot" 1; + "~*InternetMeasurement" 1; + "~*Sogou" 1; + "~*IonCrawl" 1; + "~*GeedoProductSearch" 1; + "~*Go-http-client" 1; + "~*CensysInspect" 1; + "~*paloaltonetworks" 1; + "~*paloalto" 1; + "~*sindresorhus" 1; + "~*Mozlila" 1; + "~*CFNetwork" 1; + "~*CCBot" 1; + "~*panscient.com" 1; + "~*Bidtellect" 1; + "~*ZoominfoBot" 1; + "~*ALittle" 1; + "~*Wappalyzer" 1; + "~*Dalvik" 1; + "~*Salesforce" 1; + "~*python-requests" 1; + "~*FriendlyCrawler" 1; + "~*facebookscraper" 1; + "~*security_txt_analysis" 1; + "~*security" 1; + "~*analysis" 1; + "~*RepoLookoutBot" 1; + "~*DuckDuckBot" 1; + "~*HatenaBookmark" 1; + "~*Proxmox" 1; + "~*Searcherweb" 1; + "~*python" 1; + "~*aiohttp" 1; + "~*Jigsaw" 1; + "~*Gaisbot" 1; + "~*UCWEB" 1; + "~*ddg_android" 1; + "~*Lynx" 1; + "~*backlink" 1; + "~*okhttp" 1; + "~*2ip" 1; + "~*ELinks" 1; + "~*UP.Link" 1; + "~*Scrapy" 1; + "~*Ruby" 1; + "~*libnup" 1; + "~*Facebot" 1; + "~*SearchExpress" 1; + "~*TurnitinBot" 1; + "~*Links" 1; + "~*AdsBot-Google" 1; + "~*DoCoMo" 1; + "~*Baiduspider" 1; + "~*iTunes" 1; + "~*Screaming" 1; + "~*SEO" 1; + "~*spider" 1; + "~*SonyEricssonT" 1; + "~*Configuration" 1; + "~*MJ12bot" 1; + "~*centuryb" 1; + "~*@" 1; + "~*360Spider" 1; + "~*404checker" 1; + "~*404enemy" 1; + "~*80legs" 1; + "~*Abonti" 1; + "~*Aboundex" 1; + "~*Aboundexbot" 1; + "~*Acunetix" 1; + "~*ADmantX" 1; + "~*AfD-Verbotsverfahren" 1; + "~*AIBOT" 1; + "~*AiHitBot" 1; + "~*Aipbot" 1; + "~*Alexibot" 1; + "~*Alligator" 1; + "~*AllSubmitter" 1; + "~*AlphaBot" 1; + "~*Anarchie" 1; + "~*Ankit" 1; + "~*Anthill" 1; + "~*Apexoo" 1; + "~*arquivo.pt" 1; + "~*arquivo-web-crawler" 1; + "~*Aspiegel" 1; + "~*ASPSeek" 1; + "~*Asterias" 1; + "~*Attach" 1; + "~*autoemailspider" 1; + "~*AwarioRssBot" 1; + "~*AwarioSmartBot" 1; + "~*BackDoorBot" 1; + "~*Backlink-Ceck" 1; + "~*backlink-check" 1; + "~*BacklinkCrawler" 1; + "~*BackStreet" 1; + "~*BackWeb" 1; + "~*Badass" 1; + "~*Bandit" 1; + "~*Barkrowler" 1; + "~*BatchFTP" 1; + "~*Battleztar\ Bazinga" 1; + "~*BBBike" 1; + "~*BDCbot" 1; + "~*BDFetch" 1; + "~*BetaBot" 1; + "~*Bigfoot" 1; + "~*Bitacle" 1; + "~*Blackboard" 1; + "~*Black\ Hole" 1; + "~*BlackWidow" 1; + "~*BLEXBot" 1; + "~*Blow" 1; + "~*BlowFish" 1; + "~*Boardreader" 1; + "~*Bolt" 1; + "~*BotALot" 1; + "~*Brandprotect" 1; + "~*Brandwatch" 1; + "~*Buck" 1; + "~*Buddy" 1; + "~*BuiltBotTough" 1; + "~*BuiltWith" 1; + "~*Bullseye" 1; + "~*BunnySlippers" 1; + "~*BuzzSumo" 1; + "~*Calculon" 1; + "~*CATExplorador" 1; + "~*CazoodleBot" 1; + "~*CCBot" 1; + "~*Cegbfeieh" 1; + "~*CensysInspect" 1; + "~*check1.exe" 1; + "~*CheeseBot" 1; + "~*CherryPicker" 1; + "~*CheTeam" 1; + "~*ChinaClaw" 1; + "~*Chlooe" 1; + "~*Claritybot" 1; + "~*Cliqzbot" 1; + "~*Cloud\ mapping" 1; + "~*coccocbot-web" 1; + "~*Cocolyzebot" 1; + "~*CODE87" 1; + "~*Cogentbot" 1; + "~*cognitiveseo" 1; + "~*Collector" 1; + "~*colly" 1; + "~*com.plumanalytics" 1; + "~*Copier" 1; + "~*CopyRightCheck" 1; + "~*Copyscape" 1; + "~*Cosmos" 1; + "~*Craftbot" 1; + "~*crawler4j" 1; + "~*crawler.feedback" 1; + "~*crawl.sogou.com" 1; + "~*CrazyWebCrawler" 1; + "~*Crescent" 1; + "~*CrunchBot" 1; + "~*CSHttp" 1; + "~*Curious" 1; + "~*Custo" 1; + "~*CyotekWebCopy" 1; + "~*DatabaseDriverMysqli" 1; + "~*DataCha0s" 1; + "~*DBLBot" 1; + "~*demandbase-bot" 1; + "~*Demon" 1; + "~*Deusu" 1; + "~*Devil" 1; + "~*Digincore" 1; + "~*DigitalPebble" 1; + "~*DIIbot" 1; + "~*Dirbuster" 1; + "~*Disco" 1; + "~*Discobot" 1; + "~*Discoverybot" 1; + "~*Dispatch" 1; + "~*DittoSpyder" 1; + "~*DnyzBot" 1; + "~*DomainAppender" 1; + "~*DomainCrawler" 1; + "~*DomainSigmaCrawler" 1; + "~*Domains\ Project" 1; + "~*domainsproject.org" 1; + "~*DomainStatsBot" 1; + "~*Dotbot" 1; + "~*Download\ Wonder" 1; + "~*Dragonfly" 1; + "~*Drip" 1; + "~*DSearch" 1; + "~*DTS\ Agent" 1; + "~*EasyDL" 1; + "~*Ebingbong" 1; + "~*eCatch" 1; + "~*ECCP/1.0" 1; + "~*Ecxi" 1; + "~*EirGrabber" 1; + "~*EMail\ Siphon" 1; + "~*EMail\ Wolf" 1; + "~*EroCrawler" 1; + "~*evc-batch" 1; + "~*Evil" 1; + "~*Exabot" 1; + "~*Express\ WebPictures" 1; + "~*ExtLinksBot" 1; + "~*Extractor" 1; + "~*ExtractorPro" 1; + "~*Extreme\ Picture\ Finder" 1; + "~*EyeNetIE" 1; + "~*Ezooms" 1; + "~*facebookscraper" 1; + "~*FDM" 1; + "~*FemtosearchBot" 1; + "~*FHscan" 1; + "~*Fimap" 1; + "~*Firefox/7.0" 1; + "~*FlashGet" 1; + "~*Flunky" 1; + "~*Foobot" 1; + "~*Freeuploader" 1; + "~*FrontPage" 1; + "~*FyberSpider" 1; + "~*Fyrebot" 1; + "~*GalaxyBot" 1; + "~*Genieo" 1; + "~*GermCrawler" 1; + "~*Getintent" 1; + "~*GetRight" 1; + "~*GetWeb" 1; + "~*Gigablast" 1; + "~*Gigabot" 1; + "~*G-i-g-a-b-o-t" 1; + "~*Go-Ahead-Got-It" 1; + "~*gobuster" 1; + "~*Gotit" 1; + "~*GoZilla" 1; + "~*Go!Zilla" 1; + "~*Grabber" 1; + "~*GrabNet" 1; + "~*Grafula" 1; + "~*GrapeFX" 1; + "~*GridBot" 1; + "~*GT::WWW" 1; + "~*Haansoft" 1; + "~*HaosouSpider" 1; + "~*Harvest" 1; + "~*Havij" 1; + "~*HEADMasterSEO" 1; + "~*heritrix" 1; + "~*Heritrix" 1; + "~*Hloader" 1; + "~*HMView" 1; + "~*HTMLparser" 1; + "~*HTTP::Lite" 1; + "~*HTTrack" 1; + "~*Humanlinks" 1; + "~*HybridBot" 1; + "~*Iblog" 1; + "~*IDBot" 1; + "~*IDBTE4M" 1; + "~*Id-search" 1; + "~*IlseBot" 1; + "~*Image\ Fetch" 1; + "~*Image\ Sucker" 1; + "~*IndeedBot" 1; + "~*Indy\ Library" 1; + "~*InfoNaviRobot" 1; + "~*InfoTekies" 1; + "~*instabid" 1; + "~*Intelliseek" 1; + "~*InterGET" 1; + "~*Internet\ Ninja" 1; + "~*InternetSeer" 1; + "~*internetVista\ monitor" 1; + "~*ips-agent" 1; + "~*Iria" 1; + "~*IRLbot" 1; + "~*isitwp.com" 1; + "~*Iskanie" 1; + "~*IstellaBot" 1; + "~*JamesBOT" 1; + "~*Jbrofuzz" 1; + "~*JennyBot" 1; + "~*JetCar" 1; + "~*Jetty" 1; + "~*JikeSpider" 1; + "~*JOC\ Web\ Spider" 1; + "~*Joomla" 1; + "~*Jorgee" 1; + "~*JustView" 1; + "~*Jyxobot" 1; + "~*Kenjin\ Spider" 1; + "~*Keyword\ Density" 1; + "~*Kinza" 1; + "~*Kozmosbot" 1; + "~*Lanshanbot" 1; + "~*Larbin" 1; + "~*LeechFTP" 1; + "~*LeechGet" 1; + "~*LexiBot" 1; + "~*Lftp" 1; + "~*LibWeb" 1; + "~*Libwhisker" 1; + "~*LieBaoFast" 1; + "~*Lightspeedsystems" 1; + "~*Likse" 1; + "~*Linkbot" 1; + "~*Linkdexbot" 1; + "~*LinkextractorPro" 1; + "~*LinkpadBot" 1; + "~*LinkScan" 1; + "~*LinksManager" 1; + "~*LinkWalker" 1; + "~*LinqiaMetadataDownloaderBot" 1; + "~*LinqiaRSSBot" 1; + "~*LinqiaScrapeBot" 1; + "~*Lipperhey" 1; + "~*Lipperhey\ Spider" 1; + "~*Litemage_walker" 1; + "~*Lmspider" 1; + "~*LNSpiderguy" 1; + "~*Ltx71" 1; + "~*lwp-request" 1; + "~*LWP::Simple" 1; + "~*lwp-trivial" 1; + "~*Magnet" 1; + "~*Mag-Net" 1; + "~*magpie-crawler" 1; + "~*Majestic12" 1; + "~*Majestic-SEO" 1; + "~*Majestic\ SEO" 1; + "~*MarkMonitor" 1; + "~*MarkWatch" 1; + "~*Masscan" 1; + "~*masscan" 1; + "~*Mass\ Downloader" 1; + "~*Mata\ Hari" 1; + "~*MauiBot" 1; + "~*Mb2345Browser" 1; + "~*meanpathbot" 1; + "~*Meanpathbot" 1; + "~*MeanPath\ Bot" 1; + "~*Mediatoolkitbot" 1; + "~*mediawords" 1; + "~*MegaIndex.ru" 1; + "~*Metauri" 1; + "~*MFC_Tear_Sample" 1; + "~*Microsoft\ Data\ Access" 1; + "~*Microsoft\ URL\ Control" 1; + "~*MIDown\ tool" 1; + "~*MIIxpc" 1; + "~*Mister\ PiX" 1; + "~*MJ12bot" 1; + "~*Mojeek" 1; + "~*Mojolicious" 1; + "~*Morfeus\ Fucking\ Scanner" 1; + "~*Mozlila" 1; + "~*MQQBrowser" 1; + "~*Mr.4x3" 1; + "~*MSFrontPage" 1; + "~*MSIECrawler" 1; + "~*Msrabot" 1; + "~*muhstik-scan" 1; + "~*Musobot" 1; + "~*Name\ Intelligence" 1; + "~*Nameprotect" 1; + "~*Navroad" 1; + "~*NearSite" 1; + "~*Needle" 1; + "~*Nessus" 1; + "~*NetAnts" 1; + "~*Netcraft" 1; + "~*netEstate\ NE\ Crawler" 1; + "~*NetLyzer" 1; + "~*NetMechanic" 1; + "~*NetSpider" 1; + "~*Nettrack" 1; + "~*Net\ Vampire" 1; + "~*Netvibes" 1; + "~*NetZIP" 1; + "~*NextGenSearchBot" 1; + "~*Nibbler" 1; + "~*NICErsPRO" 1; + "~*Niki-bot" 1; + "~*Nikto" 1; + "~*NimbleCrawler" 1; + "~*Nimbostratus" 1; + "~*Ninja" 1; + "~*Nuclei" 1; + "~*Nmap" 1; + "~*NPbot" 1; + "~*Nutch" 1; + "~*oBot" 1; + "~*Octopus" 1; + "~*Offline\ Explorer" 1; + "~*Offline\ Navigator" 1; + "~*OnCrawl" 1; + "~*Openfind" 1; + "~*OpenLinkProfiler" 1; + "~*Openvas" 1; + "~*OpenVAS" 1; + "~*OPPO A33" 1; + "~*OrangeBot" 1; + "~*OrangeSpider" 1; + "~*OutclicksBot" 1; + "~*OutfoxBot" 1; + "~*PageAnalyzer" 1; + "~*Page\ Analyzer" 1; + "~*PageGrabber" 1; + "~*page\ scorer" 1; + "~*PageScorer" 1; + "~*Pandalytics" 1; + "~*Panscient" 1; + "~*Papa\ Foto" 1; + "~*Pavuk" 1; + "~*pcBrowser" 1; + "~*PECL::HTTP" 1; + "~*PeoplePal" 1; + "~*Petalbot" 1; + "~*PHPCrawl" 1; + "~*Picscout" 1; + "~*Picsearch" 1; + "~*PictureFinder" 1; + "~*Pimonster" 1; + "~*Pi-Monster" 1; + "~*Pixray" 1; + "~*PleaseCrawl" 1; + "~*plumanalytics" 1; + "~*Pockey" 1; + "~*POE-Component-Client-HTTP" 1; + "~*polaris\ version" 1; + "~*Probethenet" 1; + "~*ProPowerBot" 1; + "~*ProWebWalker" 1; + "~*Psbot" 1; + "~*Pump" 1; + "~*PxBroker" 1; + "~*PyCurl" 1; + "~*QueryN\ Metasearch" 1; + "~*Quick-Crawler" 1; + "~*RankActive" 1; + "~*RankActiveLinkBot" 1; + "~*RankFlex" 1; + "~*RankingBot" 1; + "~*RankingBot2" 1; + "~*Rankivabot" 1; + "~*RankurBot" 1; + "~*RealDownload" 1; + "~*Reaper" 1; + "~*RebelMouse" 1; + "~*Recorder" 1; + "~*RedesScrapy" 1; + "~*ReGet" 1; + "~*RepoMonkey" 1; + "~*Ripper" 1; + "~*RocketCrawler" 1; + "~*Rogerbot" 1; + "~*RSSingBot" 1; + "~*s1z.ru" 1; + "~*SalesIntelligent" 1; + "~*satoristudio.net" 1; + "~*SBIder" 1; + "~*ScanAlert" 1; + "~*Scanbot" 1; + "~*scan.lol" 1; + "~*ScoutJet" 1; + "~*Scrapy" 1; + "~*Screaming" 1; + "~*ScreenerBot" 1; + "~*Searchestate" 1; + "~*SearchmetricsBot" 1; + "~*SentiBot" 1; + "~*SEOkicks" 1; + "~*SEOkicks-Robot" 1; + "~*SEOlyticsCrawler" 1; + "~*Seomoz" 1; + "~*SEOprofiler" 1; + "~*seoscanners" 1; + "~*SeoSiteCheckup" 1; + "~*SEOstats" 1; + "~*serpstatbot" 1; + "~*sexsearcher" 1; + "~*Shodan" 1; + "~*Siphon" 1; + "~*SISTRIX" 1; + "~*Sitebeam" 1; + "~*SiteCheckerBotCrawler" 1; + "~*sitechecker.pro" 1; + "~*SiteExplorer" 1; + "~*Siteimprove" 1; + "~*SiteLockSpider" 1; + "~*SiteSnagger" 1; + "~*SiteSucker" 1; + "~*Site\ Sucker" 1; + "~*Sitevigil" 1; + "~*SlySearch" 1; + "~*SmartDownload" 1; + "~*SMTBot" 1; + "~*Snake" 1; + "~*Snapbot" 1; + "~*Snoopy" 1; + "~*SocialRankIOBot" 1; + "~*Sociscraper" 1; + "~*sogouspider" 1; + "~*Sogou\ web\ spider" 1; + "~*Sosospider" 1; + "~*Sottopop" 1; + "~*SpaceBison" 1; + "~*Spammen" 1; + "~*SpankBot" 1; + "~*Spanner" 1; + "~*sp_auditbot" 1; + "~*Spbot" 1; + "~*Spinn3r" 1; + "~*SputnikBot" 1; + "~*spyfu" 1; + "~*Sqlmap" 1; + "~*Sqlworm" 1; + "~*Sqworm" 1; + "~*Steeler" 1; + "~*Stripper" 1; + "~*Sucker" 1; + "~*Sucuri" 1; + "~*SuperBot" 1; + "~*SuperHTTP" 1; + "~*Surfbot" 1; + "~*SurveyBot" 1; + "~*Suzuran" 1; + "~*Swiftbot" 1; + "~*sysscan" 1; + "~*Szukacz" 1; + "~*T0PHackTeam" 1; + "~*T8Abot" 1; + "~*tAkeOut" 1; + "~*Teleport" 1; + "~*TeleportPro" 1; + "~*Telesoft" 1; + "~*Telesphoreo" 1; + "~*Telesphorep" 1; + "~*The\ Intraformant" 1; + "~*TheNomad" 1; + "~*Thumbor" 1; + "~*TightTwatBot" 1; + "~*Titan" 1; + "~*Toata" 1; + "~*Toweyabot" 1; + "~*Tracemyfile" 1; + "~*Trendiction" 1; + "~*Trendictionbot" 1; + "~*trendiction.com" 1; + "~*trendiction.de" 1; + "~*True_Robot" 1; + "~*Turingos" 1; + "~*Turnitin" 1; + "~*TurnitinBot" 1; + "~*TwengaBot" 1; + "~*Twice" 1; + "~*Typhoeus" 1; + "~*UnisterBot" 1; + "~*Upflow" 1; + "~*URLy.Warning" 1; + "~*URLy\ Warning" 1; + "~*Vacuum" 1; + "~*Vagabondo" 1; + "~*VB\ Project" 1; + "~*VCI" 1; + "~*VelenPublicWebCrawler" 1; + "~*VeriCiteCrawler" 1; + "~*VidibleScraper" 1; + "~*Virusdie" 1; + "~*VoidEYE" 1; + "~*Voil" 1; + "~*Voltron" 1; + "~*Wallpapers/3.0" 1; + "~*WallpapersHD" 1; + "~*WASALive-Bot" 1; + "~*WBSearchBot" 1; + "~*Webalta" 1; + "~*WebAuto" 1; + "~*Web\ Auto" 1; + "~*WebBandit" 1; + "~*WebCollage" 1; + "~*Web\ Collage" 1; + "~*WebCopier" 1; + "~*WEBDAV" 1; + "~*WebEnhancer" 1; + "~*Web\ Enhancer" 1; + "~*WebFetch" 1; + "~*Web\ Fetch" 1; + "~*WebFuck" 1; + "~*Web\ Fuck" 1; + "~*WebGo\ IS" 1; + "~*WebImageCollector" 1; + "~*WebLeacher" 1; + "~*WebmasterWorldForumBot" 1; + "~*webmeup-crawler" 1; + "~*WebPix" 1; + "~*Web\ Pix" 1; + "~*WebReaper" 1; + "~*WebSauger" 1; + "~*Web\ Sauger" 1; + "~*Webshag" 1; + "~*WebsiteExtractor" 1; + "~*WebsiteQuester" 1; + "~*Website\ Quester" 1; + "~*Webster" 1; + "~*WebStripper" 1; + "~*WebSucker" 1; + "~*Web\ Sucker" 1; + "~*WebWhacker" 1; + "~*WebZIP" 1; + "~*WeSEE" 1; + "~*Whack" 1; + "~*Whacker" 1; + "~*Whatweb" 1; + "~*Who.is\ Bot" 1; + "~*Widow" 1; + "~*WinHTTrack" 1; + "~*WiseGuys\ Robot" 1; + "~*WISENutbot" 1; + "~*Wonderbot" 1; + "~*Woobot" 1; + "~*Wotbox" 1; + "~*Wprecon" 1; + "~*WPScan" 1; + "~*WWW-Collector-E" 1; + "~*WWW-Mechanize" 1; + "~*WWW::Mechanize" 1; + "~*WWWOFFLE" 1; + "~*x09Mozilla" 1; + "~*x22Mozilla" 1; + "~*Xaldon_WebSpider" 1; + "~*Xaldon\ WebSpider" 1; + "~*Xenu" 1; + "~*xpymep1.exe" 1; + "~*YoudaoBot" 1; + "~*Zade" 1; + "~*Zauba" 1; + "~*zauba.io" 1; + "~*Zermelo" 1; + "~*Zeus" 1; + "~*zgrab" 1; + "~*Zitebot" 1; + "~*ZmEu" 1; + "~*ZoomBot" 1; + "~*ZoominfoBot" 1; + "~*ZumBot" 1; + "~*ZyBorg" 1; + # Add other bot user agents as needed +} \ No newline at end of file