cefd9cf283
* Add crawler blocking for fallback: true * update test * Update check * Add note to docs * use browser agent for non-crawler test
22 lines
912 B
TypeScript
22 lines
912 B
TypeScript
import { BLOCKED_PAGES } from '../shared/lib/constants'
|
|
|
|
export function isBlockedPage(pathname: string): boolean {
|
|
return BLOCKED_PAGES.includes(pathname)
|
|
}
|
|
|
|
export function cleanAmpPath(pathname: string): string {
|
|
if (pathname.match(/\?amp=(y|yes|true|1)/)) {
|
|
pathname = pathname.replace(/\?amp=(y|yes|true|1)&?/, '?')
|
|
}
|
|
if (pathname.match(/&=(y|yes|true|1)/)) {
|
|
pathname = pathname.replace(/&=(y|yes|true|1)/, '')
|
|
}
|
|
pathname = pathname.replace(/\?$/, '')
|
|
return pathname
|
|
}
|
|
|
|
export function isBot(userAgent: string): boolean {
|
|
return /Googlebot|Mediapartners-Google|AdsBot-Google|googleweblight|Storebot-Google|Bingbot|BingPreview|Slurp|DuckDuckBot|baiduspider|yandex|sogou|LinkedInBot|bitlybot|tumblr|vkShare|quora link preview|facebookexternalhit|facebookcatalog|Twitterbot|applebot|redditbot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|ia_archiver/i.test(
|
|
userAgent
|
|
)
|
|
}
|