Rename to hkt.sh

This commit is contained in:
mango
2026-03-21 01:10:53 +08:00
parent 76a263d0f9
commit 8f1171fe99
6676 changed files with 1724268 additions and 0 deletions

21
node_modules/puppeteer-extra-plugin-stealth/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2019 berstend <github@berstend.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -0,0 +1,28 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
/**
* Minimal stealth plugin template, not being used. :-)
*
* Feel free to copy this folder as the basis for additional detection evasion plugins.
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/_template'
}
async onPageCreated(page) {
await page.evaluateOnNewDocument(() => {
console.debug('hello world')
})
}
}
module.exports = function(pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,19 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_template/index.js#L10-L24)
- `opts` (optional, default `{}`)
**Extends: PuppeteerExtraPlugin**
Minimal stealth plugin template, not being used. :-)
Feel free to copy this folder as the basis for additional detection evasion plugins.
---

View File

@@ -0,0 +1,583 @@
/**
* A set of shared utility functions specifically for the purpose of modifying native browser APIs without leaving traces.
*
* Meant to be passed down in puppeteer and used in the context of the page (everything in here runs in NodeJS as well as a browser).
*
* Note: If for whatever reason you need to use this outside of `puppeteer-extra`:
* Just remove the `module.exports` statement at the very bottom, the rest can be copy pasted into any browser context.
*
* Alternatively take a look at the `extract-stealth-evasions` package to create a finished bundle which includes these utilities.
*
*/
const utils = {}
utils.init = () => {
utils.preloadCache()
}
/**
* Wraps a JS Proxy Handler and strips it's presence from error stacks, in case the traps throw.
*
* The presence of a JS Proxy can be revealed as it shows up in error stack traces.
*
* @param {object} handler - The JS Proxy handler to wrap
*/
utils.stripProxyFromErrors = (handler = {}) => {
const newHandler = {
setPrototypeOf: function (target, proto) {
if (proto === null)
throw new TypeError('Cannot convert object to primitive value')
if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {
throw new TypeError('Cyclic __proto__ value')
}
return Reflect.setPrototypeOf(target, proto)
}
}
// We wrap each trap in the handler in a try/catch and modify the error stack if they throw
const traps = Object.getOwnPropertyNames(handler)
traps.forEach(trap => {
newHandler[trap] = function () {
try {
// Forward the call to the defined proxy handler
return handler[trap].apply(this, arguments || [])
} catch (err) {
// Stack traces differ per browser, we only support chromium based ones currently
if (!err || !err.stack || !err.stack.includes(`at `)) {
throw err
}
// When something throws within one of our traps the Proxy will show up in error stacks
// An earlier implementation of this code would simply strip lines with a blacklist,
// but it makes sense to be more surgical here and only remove lines related to our Proxy.
// We try to use a known "anchor" line for that and strip it with everything above it.
// If the anchor line cannot be found for some reason we fall back to our blacklist approach.
const stripWithBlacklist = (stack, stripFirstLine = true) => {
const blacklist = [
`at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply
`at Object.${trap} `, // e.g. Object.get or Object.apply
`at Object.newHandler.<computed> [as ${trap}] ` // caused by this very wrapper :-)
]
return (
err.stack
.split('\n')
// Always remove the first (file) line in the stack (guaranteed to be our proxy)
.filter((line, index) => !(index === 1 && stripFirstLine))
// Check if the line starts with one of our blacklisted strings
.filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))
.join('\n')
)
}
const stripWithAnchor = (stack, anchor) => {
const stackArr = stack.split('\n')
anchor = anchor || `at Object.newHandler.<computed> [as ${trap}] ` // Known first Proxy line in chromium
const anchorIndex = stackArr.findIndex(line =>
line.trim().startsWith(anchor)
)
if (anchorIndex === -1) {
return false // 404, anchor not found
}
// Strip everything from the top until we reach the anchor line
// Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)
stackArr.splice(1, anchorIndex)
return stackArr.join('\n')
}
// Special cases due to our nested toString proxies
err.stack = err.stack.replace(
'at Object.toString (',
'at Function.toString ('
)
if ((err.stack || '').includes('at Function.toString (')) {
err.stack = stripWithBlacklist(err.stack, false)
throw err
}
// Try using the anchor method, fallback to blacklist if necessary
err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)
throw err // Re-throw our now sanitized error
}
}
})
return newHandler
}
/**
* Strip error lines from stack traces until (and including) a known line the stack.
*
* @param {object} err - The error to sanitize
* @param {string} anchor - The string the anchor line starts with
*/
utils.stripErrorWithAnchor = (err, anchor) => {
const stackArr = err.stack.split('\n')
const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))
if (anchorIndex === -1) {
return err // 404, anchor not found
}
// Strip everything from the top until we reach the anchor line (remove anchor line as well)
// Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)
stackArr.splice(1, anchorIndex)
err.stack = stackArr.join('\n')
return err
}
/**
* Replace the property of an object in a stealthy way.
*
* Note: You also want to work on the prototype of an object most often,
* as you'd otherwise leave traces (e.g. showing up in Object.getOwnPropertyNames(obj)).
*
* @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty
*
* @example
* replaceProperty(WebGLRenderingContext.prototype, 'getParameter', { value: "alice" })
* // or
* replaceProperty(Object.getPrototypeOf(navigator), 'languages', { get: () => ['en-US', 'en'] })
*
* @param {object} obj - The object which has the property to replace
* @param {string} propName - The property name to replace
* @param {object} descriptorOverrides - e.g. { value: "alice" }
*/
utils.replaceProperty = (obj, propName, descriptorOverrides = {}) => {
return Object.defineProperty(obj, propName, {
// Copy over the existing descriptors (writable, enumerable, configurable, etc)
...(Object.getOwnPropertyDescriptor(obj, propName) || {}),
// Add our overrides (e.g. value, get())
...descriptorOverrides
})
}
/**
* Preload a cache of function copies and data.
*
* For a determined enough observer it would be possible to overwrite and sniff usage of functions
* we use in our internal Proxies, to combat that we use a cached copy of those functions.
*
* Note: Whenever we add a `Function.prototype.toString` proxy we should preload the cache before,
* by executing `utils.preloadCache()` before the proxy is applied (so we don't cause recursive lookups).
*
* This is evaluated once per execution context (e.g. window)
*/
utils.preloadCache = () => {
if (utils.cache) {
return
}
utils.cache = {
// Used in our proxies
Reflect: {
get: Reflect.get.bind(Reflect),
apply: Reflect.apply.bind(Reflect)
},
// Used in `makeNativeString`
nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`
}
}
/**
* Utility function to generate a cross-browser `toString` result representing native code.
*
* There's small differences: Chromium uses a single line, whereas FF & Webkit uses multiline strings.
* To future-proof this we use an existing native toString result as the basis.
*
* The only advantage we have over the other team is that our JS runs first, hence we cache the result
* of the native toString result once, so they cannot spoof it afterwards and reveal that we're using it.
*
* @example
* makeNativeString('foobar') // => `function foobar() { [native code] }`
*
* @param {string} [name] - Optional function name
*/
utils.makeNativeString = (name = '') => {
return utils.cache.nativeToStringStr.replace('toString', name || '')
}
/**
* Helper function to modify the `toString()` result of the provided object.
*
* Note: Use `utils.redirectToString` instead when possible.
*
* There's a quirk in JS Proxies that will cause the `toString()` result to differ from the vanilla Object.
* If no string is provided we will generate a `[native code]` thing based on the name of the property object.
*
* @example
* patchToString(WebGLRenderingContext.prototype.getParameter, 'function getParameter() { [native code] }')
*
* @param {object} obj - The object for which to modify the `toString()` representation
* @param {string} str - Optional string used as a return value
*/
utils.patchToString = (obj, str = '') => {
const handler = {
apply: function (target, ctx) {
// This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + ""`
if (ctx === Function.prototype.toString) {
return utils.makeNativeString('toString')
}
// `toString` targeted at our proxied Object detected
if (ctx === obj) {
// We either return the optional string verbatim or derive the most desired result automatically
return str || utils.makeNativeString(obj.name)
}
// Check if the toString protype of the context is the same as the global prototype,
// if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case
const hasSameProto = Object.getPrototypeOf(
Function.prototype.toString
).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins
if (!hasSameProto) {
// Pass the call on to the local Function.prototype.toString instead
return ctx.toString()
}
return target.call(ctx)
}
}
const toStringProxy = new Proxy(
Function.prototype.toString,
utils.stripProxyFromErrors(handler)
)
utils.replaceProperty(Function.prototype, 'toString', {
value: toStringProxy
})
}
/**
* Make all nested functions of an object native.
*
* @param {object} obj
*/
utils.patchToStringNested = (obj = {}) => {
return utils.execRecursively(obj, ['function'], utils.patchToString)
}
/**
* Redirect toString requests from one object to another.
*
* @param {object} proxyObj - The object that toString will be called on
* @param {object} originalObj - The object which toString result we wan to return
*/
utils.redirectToString = (proxyObj, originalObj) => {
const handler = {
apply: function (target, ctx) {
// This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + ""`
if (ctx === Function.prototype.toString) {
return utils.makeNativeString('toString')
}
// `toString` targeted at our proxied Object detected
if (ctx === proxyObj) {
const fallback = () =>
originalObj && originalObj.name
? utils.makeNativeString(originalObj.name)
: utils.makeNativeString(proxyObj.name)
// Return the toString representation of our original object if possible
return originalObj + '' || fallback()
}
if (typeof ctx === 'undefined' || ctx === null) {
return target.call(ctx)
}
// Check if the toString protype of the context is the same as the global prototype,
// if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case
const hasSameProto = Object.getPrototypeOf(
Function.prototype.toString
).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins
if (!hasSameProto) {
// Pass the call on to the local Function.prototype.toString instead
return ctx.toString()
}
return target.call(ctx)
}
}
const toStringProxy = new Proxy(
Function.prototype.toString,
utils.stripProxyFromErrors(handler)
)
utils.replaceProperty(Function.prototype, 'toString', {
value: toStringProxy
})
}
/**
* All-in-one method to replace a property with a JS Proxy using the provided Proxy handler with traps.
*
* Will stealthify these aspects (strip error stack traces, redirect toString, etc).
* Note: This is meant to modify native Browser APIs and works best with prototype objects.
*
* @example
* replaceWithProxy(WebGLRenderingContext.prototype, 'getParameter', proxyHandler)
*
* @param {object} obj - The object which has the property to replace
* @param {string} propName - The name of the property to replace
* @param {object} handler - The JS Proxy handler to use
*/
utils.replaceWithProxy = (obj, propName, handler) => {
const originalObj = obj[propName]
const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))
utils.replaceProperty(obj, propName, { value: proxyObj })
utils.redirectToString(proxyObj, originalObj)
return true
}
/**
* All-in-one method to replace a getter with a JS Proxy using the provided Proxy handler with traps.
*
* @example
* replaceGetterWithProxy(Object.getPrototypeOf(navigator), 'vendor', proxyHandler)
*
* @param {object} obj - The object which has the property to replace
* @param {string} propName - The name of the property to replace
* @param {object} handler - The JS Proxy handler to use
*/
utils.replaceGetterWithProxy = (obj, propName, handler) => {
const fn = Object.getOwnPropertyDescriptor(obj, propName).get
const fnStr = fn.toString() // special getter function string
const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))
utils.replaceProperty(obj, propName, { get: proxyObj })
utils.patchToString(proxyObj, fnStr)
return true
}
/**
* All-in-one method to replace a getter and/or setter. Functions get and set
* of handler have one more argument that contains the native function.
*
* @example
* replaceGetterSetter(HTMLIFrameElement.prototype, 'contentWindow', handler)
*
* @param {object} obj - The object which has the property to replace
* @param {string} propName - The name of the property to replace
* @param {object} handlerGetterSetter - The handler with get and/or set
* functions
* @see https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty#description
*/
utils.replaceGetterSetter = (obj, propName, handlerGetterSetter) => {
const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName)
const handler = { ...ownPropertyDescriptor }
if (handlerGetterSetter.get !== undefined) {
const nativeFn = ownPropertyDescriptor.get
handler.get = function() {
return handlerGetterSetter.get.call(this, nativeFn.bind(this))
}
utils.redirectToString(handler.get, nativeFn)
}
if (handlerGetterSetter.set !== undefined) {
const nativeFn = ownPropertyDescriptor.set
handler.set = function(newValue) {
handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this))
}
utils.redirectToString(handler.set, nativeFn)
}
Object.defineProperty(obj, propName, handler)
}
/**
* All-in-one method to mock a non-existing property with a JS Proxy using the provided Proxy handler with traps.
*
* Will stealthify these aspects (strip error stack traces, redirect toString, etc).
*
* @example
* mockWithProxy(chrome.runtime, 'sendMessage', function sendMessage() {}, proxyHandler)
*
* @param {object} obj - The object which has the property to replace
* @param {string} propName - The name of the property to replace or create
* @param {object} pseudoTarget - The JS Proxy target to use as a basis
* @param {object} handler - The JS Proxy handler to use
*/
utils.mockWithProxy = (obj, propName, pseudoTarget, handler) => {
const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))
utils.replaceProperty(obj, propName, { value: proxyObj })
utils.patchToString(proxyObj)
return true
}
/**
* All-in-one method to create a new JS Proxy with stealth tweaks.
*
* This is meant to be used whenever we need a JS Proxy but don't want to replace or mock an existing known property.
*
* Will stealthify certain aspects of the Proxy (strip error stack traces, redirect toString, etc).
*
* @example
* createProxy(navigator.mimeTypes.__proto__.namedItem, proxyHandler) // => Proxy
*
* @param {object} pseudoTarget - The JS Proxy target to use as a basis
* @param {object} handler - The JS Proxy handler to use
*/
utils.createProxy = (pseudoTarget, handler) => {
const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))
utils.patchToString(proxyObj)
return proxyObj
}
/**
* Helper function to split a full path to an Object into the first part and property.
*
* @example
* splitObjPath(`HTMLMediaElement.prototype.canPlayType`)
* // => {objName: "HTMLMediaElement.prototype", propName: "canPlayType"}
*
* @param {string} objPath - The full path to an object as dot notation string
*/
utils.splitObjPath = objPath => ({
// Remove last dot entry (property) ==> `HTMLMediaElement.prototype`
objName: objPath.split('.').slice(0, -1).join('.'),
// Extract last dot entry ==> `canPlayType`
propName: objPath.split('.').slice(-1)[0]
})
/**
* Convenience method to replace a property with a JS Proxy using the provided objPath.
*
* Supports a full path (dot notation) to the object as string here, in case that makes it easier.
*
* @example
* replaceObjPathWithProxy('WebGLRenderingContext.prototype.getParameter', proxyHandler)
*
* @param {string} objPath - The full path to an object (dot notation string) to replace
* @param {object} handler - The JS Proxy handler to use
*/
utils.replaceObjPathWithProxy = (objPath, handler) => {
const { objName, propName } = utils.splitObjPath(objPath)
const obj = eval(objName) // eslint-disable-line no-eval
return utils.replaceWithProxy(obj, propName, handler)
}
/**
* Traverse nested properties of an object recursively and apply the given function on a whitelist of value types.
*
* @param {object} obj
* @param {array} typeFilter - e.g. `['function']`
* @param {Function} fn - e.g. `utils.patchToString`
*/
utils.execRecursively = (obj = {}, typeFilter = [], fn) => {
function recurse(obj) {
for (const key in obj) {
if (obj[key] === undefined) {
continue
}
if (obj[key] && typeof obj[key] === 'object') {
recurse(obj[key])
} else {
if (obj[key] && typeFilter.includes(typeof obj[key])) {
fn.call(this, obj[key])
}
}
}
}
recurse(obj)
return obj
}
/**
* Everything we run through e.g. `page.evaluate` runs in the browser context, not the NodeJS one.
* That means we cannot just use reference variables and functions from outside code, we need to pass everything as a parameter.
*
* Unfortunately the data we can pass is only allowed to be of primitive types, regular functions don't survive the built-in serialization process.
* This utility function will take an object with functions and stringify them, so we can pass them down unharmed as strings.
*
* We use this to pass down our utility functions as well as any other functions (to be able to split up code better).
*
* @see utils.materializeFns
*
* @param {object} fnObj - An object containing functions as properties
*/
utils.stringifyFns = (fnObj = { hello: () => 'world' }) => {
// Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine
// https://github.com/feross/fromentries
function fromEntries(iterable) {
return [...iterable].reduce((obj, [key, val]) => {
obj[key] = val
return obj
}, {})
}
return (Object.fromEntries || fromEntries)(
Object.entries(fnObj)
.filter(([key, value]) => typeof value === 'function')
.map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval
)
}
/**
* Utility function to reverse the process of `utils.stringifyFns`.
* Will materialize an object with stringified functions (supports classic and fat arrow functions).
*
* @param {object} fnStrObj - An object containing stringified functions as properties
*/
utils.materializeFns = (fnStrObj = { hello: "() => 'world'" }) => {
return Object.fromEntries(
Object.entries(fnStrObj).map(([key, value]) => {
if (value.startsWith('function')) {
// some trickery is needed to make oldschool functions work :-)
return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval
} else {
// arrow functions just work
return [key, eval(value)] // eslint-disable-line no-eval
}
})
)
}
// Proxy handler templates for re-usability
utils.makeHandler = () => ({
// Used by simple `navigator` getter evasions
getterValue: value => ({
apply(target, ctx, args) {
// Let's fetch the value first, to trigger and escalate potential errors
// Illegal invocations like `navigator.__proto__.vendor` will throw here
utils.cache.Reflect.apply(...arguments)
return value
}
})
})
/**
* Compare two arrays.
*
* @param {array} array1 - First array
* @param {array} array2 - Second array
*/
utils.arrayEquals = (array1, array2) => {
if (array1.length !== array2.length) {
return false
}
for (let i = 0; i < array1.length; ++i) {
if (array1[i] !== array2[i]) {
return false
}
}
return true
}
/**
* Cache the method return according to its arguments.
*
* @param {Function} fn - A function that will be cached
*/
utils.memoize = fn => {
const cache = []
return function(...args) {
if (!cache.some(c => utils.arrayEquals(c.key, args))) {
cache.push({ key: args, value: fn.apply(this, args) })
}
return cache.find(c => utils.arrayEquals(c.key, args)).value
}
}
// --
// Stuff starting below this line is NodeJS specific.
// --
module.exports = utils

View File

@@ -0,0 +1,709 @@
const test = require('ava')
const { vanillaPuppeteer } = require('../../test/util')
const utils = require('.')
const withUtils = require('./withUtils')
/* global HTMLMediaElement WebGLRenderingContext */
test('splitObjPath: will do what it says', async t => {
const { objName, propName } = utils.splitObjPath(
'HTMLMediaElement.prototype.canPlayType'
)
t.is(objName, 'HTMLMediaElement.prototype')
t.is(propName, 'canPlayType')
})
test('makeNativeString: will do what it says', async t => {
utils.init()
t.is(utils.makeNativeString('bob'), 'function bob() { [native code] }')
t.is(
utils.makeNativeString('toString'),
'function toString() { [native code] }'
)
t.is(utils.makeNativeString(), 'function () { [native code] }')
})
test('replaceWithProxy: will work correctly', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const test1 = await withUtils(page).evaluate(utils => {
const dummyProxyHandler = {
get(target, param) {
if (param && param === 'ping') {
return 'pong'
}
return utils.cache.Reflect.get(...(arguments || []))
},
apply() {
return utils.cache.Reflect.apply(...arguments)
}
}
utils.replaceWithProxy(
HTMLMediaElement.prototype,
'canPlayType',
dummyProxyHandler
)
return {
toString: HTMLMediaElement.prototype.canPlayType.toString(),
ping: HTMLMediaElement.prototype.canPlayType.ping
}
})
t.deepEqual(test1, {
toString: 'function canPlayType() { [native code] }',
ping: 'pong'
})
})
test('replaceObjPathWithProxy: will work correctly', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const test1 = await withUtils(page).evaluate(utils => {
const dummyProxyHandler = {
get(target, param) {
if (param && param === 'ping') {
return 'pong'
}
return utils.cache.Reflect.get(...(arguments || []))
},
apply() {
return utils.cache.Reflect.apply(...arguments)
}
}
utils.replaceObjPathWithProxy(
'HTMLMediaElement.prototype.canPlayType',
dummyProxyHandler
)
return {
toString: HTMLMediaElement.prototype.canPlayType.toString(),
ping: HTMLMediaElement.prototype.canPlayType.ping
}
})
t.deepEqual(test1, {
toString: 'function canPlayType() { [native code] }',
ping: 'pong'
})
})
test('redirectToString: is battle hardened', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
// Patch all documents including iframes
await withUtils(page).evaluateOnNewDocument(utils => {
// We redirect toString calls targeted at `canPlayType` to `getParameter`,
// so if everything works correctly we expect `getParameter` as response.
const proxyObj = HTMLMediaElement.prototype.canPlayType
const originalObj = WebGLRenderingContext.prototype.getParameter
utils.redirectToString(proxyObj, originalObj)
})
await page.goto('about:blank')
const result = await withUtils(page).evaluate(utils => {
const iframe = document.createElement('iframe')
document.body.appendChild(iframe)
return {
target: {
raw: HTMLMediaElement.prototype.canPlayType + '',
rawiframe:
iframe.contentWindow.HTMLMediaElement.prototype.canPlayType + '',
raw2: HTMLMediaElement.prototype.canPlayType.toString(),
rawiframe2:
iframe.contentWindow.HTMLMediaElement.prototype.canPlayType.toString(),
direct: Function.prototype.toString.call(
HTMLMediaElement.prototype.canPlayType
),
directWithiframe: iframe.contentWindow.Function.prototype.toString.call(
HTMLMediaElement.prototype.canPlayType
),
iframeWithdirect: Function.prototype.toString.call(
iframe.contentWindow.HTMLMediaElement.prototype.canPlayType
),
iframeWithiframe: iframe.contentWindow.Function.prototype.toString.call(
iframe.contentWindow.HTMLMediaElement.prototype.canPlayType
)
},
toString: {
obj: HTMLMediaElement.prototype.canPlayType.toString + '',
objiframe:
iframe.contentWindow.HTMLMediaElement.prototype.canPlayType.toString +
'',
raw: Function.prototype.toString + '',
rawiframe: iframe.contentWindow.Function.prototype.toString + '',
direct: Function.prototype.toString.call(Function.prototype.toString),
directWithiframe: iframe.contentWindow.Function.prototype.toString.call(
Function.prototype.toString
),
iframeWithdirect: Function.prototype.toString.call(
iframe.contentWindow.Function.prototype.toString
),
iframeWithiframe: iframe.contentWindow.Function.prototype.toString.call(
iframe.contentWindow.Function.prototype.toString
)
}
}
})
t.deepEqual(result, {
target: {
raw: 'function getParameter() { [native code] }',
raw2: 'function getParameter() { [native code] }',
rawiframe: 'function getParameter() { [native code] }',
rawiframe2: 'function getParameter() { [native code] }',
direct: 'function getParameter() { [native code] }',
directWithiframe: 'function getParameter() { [native code] }',
iframeWithdirect: 'function getParameter() { [native code] }',
iframeWithiframe: 'function getParameter() { [native code] }'
},
toString: {
obj: 'function toString() { [native code] }',
objiframe: 'function toString() { [native code] }',
raw: 'function toString() { [native code] }',
rawiframe: 'function toString() { [native code] }',
direct: 'function toString() { [native code] }',
directWithiframe: 'function toString() { [native code] }',
iframeWithdirect: 'function toString() { [native code] }',
iframeWithiframe: 'function toString() { [native code] }'
}
})
})
test('redirectToString: has proper errors', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
// Patch all documents including iframes
await withUtils(page).evaluateOnNewDocument(utils => {
// We redirect toString calls targeted at `canPlayType` to `getParameter`,
// so if everything works correctly we expect `getParameter` as response.
const proxyObj = HTMLMediaElement.prototype.canPlayType
const originalObj = WebGLRenderingContext.prototype.getParameter
utils.redirectToString(proxyObj, originalObj)
})
await page.goto('about:blank')
const result = await withUtils(page).evaluate(utils => {
const evalErr = (str = '') => {
try {
// eslint-disable-next-line no-eval
return eval(str)
} catch (err) {
return err.toString()
}
}
return {
blank: evalErr(`Function.prototype.toString.apply()`),
null: evalErr(`Function.prototype.toString.apply(null)`),
undef: evalErr(`Function.prototype.toString.apply(undefined)`),
emptyObject: evalErr(`Function.prototype.toString.apply({})`)
}
})
t.deepEqual(result, {
blank:
"TypeError: Function.prototype.toString requires that 'this' be a Function",
null: "TypeError: Function.prototype.toString requires that 'this' be a Function",
undef:
"TypeError: Function.prototype.toString requires that 'this' be a Function",
emptyObject:
"TypeError: Function.prototype.toString requires that 'this' be a Function"
})
})
test('patchToString: will work correctly', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
// Test verbatim string replacement
const test1 = await withUtils(page).evaluate(utils => {
utils.patchToString(HTMLMediaElement.prototype.canPlayType, 'bob')
return HTMLMediaElement.prototype.canPlayType.toString()
})
t.is(test1, 'bob')
// Test automatic mode derived from `.name`
const test2 = await withUtils(page).evaluate(utils => {
utils.patchToString(HTMLMediaElement.prototype.canPlayType)
return HTMLMediaElement.prototype.canPlayType.toString()
})
t.is(test2, 'function canPlayType() { [native code] }')
// Make sure automatic mode derived from `.name` works with proxies
const test3 = await withUtils(page).evaluate(utils => {
HTMLMediaElement.prototype.canPlayType = new Proxy(
HTMLMediaElement.prototype.canPlayType,
{}
)
utils.patchToString(HTMLMediaElement.prototype.canPlayType)
return HTMLMediaElement.prototype.canPlayType.toString()
})
t.is(test3, 'function canPlayType() { [native code] }')
// Actually verify there's an issue when using vanilla Proxies
const test4 = await withUtils(page).evaluate(utils => {
HTMLMediaElement.prototype.canPlayType = new Proxy(
HTMLMediaElement.prototype.canPlayType,
{}
)
return HTMLMediaElement.prototype.canPlayType.toString()
})
t.is(test4, 'function () { [native code] }')
})
function toStringTest(obj) {
obj = eval(obj) // eslint-disable-line no-eval
return `
- obj.toString(): ${obj.toString()}
- obj.name: ${obj.name}
- obj.toString + "": ${obj.toString + ''}
- obj.toString.name: ${obj.toString.name}
- obj.valueOf + "": ${obj.valueOf + ''}
- obj.valueOf().name: ${obj.valueOf().name}
- Object.prototype.toString.apply(obj): ${Object.prototype.toString.apply(obj)}
- Function.prototype.toString.call(obj): ${Function.prototype.toString.call(
obj
)}
- Function.prototype.valueOf.call(obj) + "": ${
Function.prototype.valueOf.call(obj) + ''
}
- obj.toString === Function.prototype.toString: ${
obj.toString === Function.prototype.toString
}
`.trim()
}
test('patchToString: passes all toString tests', async t => {
const toStringVanilla = await (async function () {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
return page.evaluate(toStringTest, 'HTMLMediaElement.prototype.canPlayType')
})()
const toStringStealth = await (async function () {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
await withUtils(page).evaluate(utils => {
HTMLMediaElement.prototype.canPlayType = function canPlayType() {}
utils.patchToString(HTMLMediaElement.prototype.canPlayType)
})
return page.evaluate(toStringTest, 'HTMLMediaElement.prototype.canPlayType')
})()
// Check that the unmodified results are as expected
t.is(
toStringVanilla,
`
- obj.toString(): function canPlayType() { [native code] }
- obj.name: canPlayType
- obj.toString + "": function toString() { [native code] }
- obj.toString.name: toString
- obj.valueOf + "": function valueOf() { [native code] }
- obj.valueOf().name: canPlayType
- Object.prototype.toString.apply(obj): [object Function]
- Function.prototype.toString.call(obj): function canPlayType() { [native code] }
- Function.prototype.valueOf.call(obj) + "": function canPlayType() { [native code] }
- obj.toString === Function.prototype.toString: true
`.trim()
)
// Make sure our customizations leave no trace
t.is(toStringVanilla, toStringStealth)
})
test('patchToString: passes stack trace tests', async t => {
const toStringStackTrace = () => {
try {
Object.create(
Object.getOwnPropertyDescriptor(Function.prototype, 'toString').get
).toString()
} catch (err) {
return err.stack.split('\n').slice(0, 2).join('|')
}
return 'error not thrown'
}
const toStringVanilla = await (async function () {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
return page.evaluate(toStringStackTrace)
})()
const toStringStealth = await (async function () {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
await withUtils(page).evaluate(utils => {
HTMLMediaElement.prototype.canPlayType = function canPlayType() {}
utils.patchToString(HTMLMediaElement.prototype.canPlayType)
})
return page.evaluate(toStringStackTrace)
})()
// Check that the unmodified results are as expected
t.is(
toStringVanilla,
`TypeError: Object prototype may only be an Object or null: undefined| at Function.create (<anonymous>)`.trim()
)
// Make sure our customizations leave no trace
t.is(toStringVanilla, toStringStealth)
})
test('patchToString: vanilla has iframe issues', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
// Only patch the main window
const result = await withUtils(page).evaluate(utils => {
utils.patchToString(HTMLMediaElement.prototype.canPlayType, 'bob')
const iframe = document.createElement('iframe')
document.body.appendChild(iframe)
return {
direct: Function.prototype.toString.call(
HTMLMediaElement.prototype.canPlayType
),
directWithiframe: iframe.contentWindow.Function.prototype.toString.call(
HTMLMediaElement.prototype.canPlayType
),
iframeWithdirect: Function.prototype.toString.call(
iframe.contentWindow.HTMLMediaElement.prototype.canPlayType
),
iframeWithiframe: iframe.contentWindow.Function.prototype.toString.call(
iframe.contentWindow.HTMLMediaElement.prototype.canPlayType
)
}
})
t.deepEqual(result, {
direct: 'bob',
directWithiframe: 'function canPlayType() { [native code] }',
iframeWithdirect: 'function canPlayType() { [native code] }',
iframeWithiframe: 'function canPlayType() { [native code] }'
})
})
test('patchToString: stealth has no iframe issues', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
// Patch all documents including iframes
await withUtils(page).evaluateOnNewDocument(utils => {
utils.patchToString(HTMLMediaElement.prototype.canPlayType, 'alice')
})
await page.goto('about:blank')
const result = await withUtils(page).evaluate(utils => {
const iframe = document.createElement('iframe')
document.body.appendChild(iframe)
return {
direct: Function.prototype.toString.call(
HTMLMediaElement.prototype.canPlayType
),
directWithiframe: iframe.contentWindow.Function.prototype.toString.call(
HTMLMediaElement.prototype.canPlayType
),
iframeWithdirect: Function.prototype.toString.call(
iframe.contentWindow.HTMLMediaElement.prototype.canPlayType
),
iframeWithiframe: iframe.contentWindow.Function.prototype.toString.call(
iframe.contentWindow.HTMLMediaElement.prototype.canPlayType
)
}
})
t.deepEqual(result, {
direct: 'alice',
directWithiframe: 'alice',
iframeWithdirect: 'alice',
iframeWithiframe: 'alice'
})
})
test('stripProxyFromErrors: will work correctly', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await withUtils(page).evaluate(utils => {
const getStack = prop => {
try {
prop.caller() // Will throw (HTMLMediaElement.prototype.canPlayType.caller)
return false
} catch (err) {
return err.stack
}
}
/** We need traps to show up in the error stack */
const dummyProxyHandler = {
get() {
return utils.cache.Reflect.get(...(arguments || []))
},
apply() {
return utils.cache.Reflect.apply(...arguments)
}
}
const vanillaProxy = new Proxy(
HTMLMediaElement.prototype.canPlayType,
dummyProxyHandler
)
const stealthProxy = new Proxy(
HTMLMediaElement.prototype.canPlayType,
utils.stripProxyFromErrors(dummyProxyHandler)
)
const stacks = {
vanilla: getStack(HTMLMediaElement.prototype.canPlayType),
vanillaProxy: getStack(vanillaProxy),
stealthProxy: getStack(stealthProxy)
}
return stacks
})
// Check that the untouched stuff behaves as expected
t.true(results.vanilla.includes(`TypeError: 'caller'`))
t.false(results.vanilla.includes(`at Object.get`))
// Regression test: Make sure vanilla JS Proxies leak the stack trace
t.true(results.vanillaProxy.includes(`TypeError: 'caller'`))
t.true(results.vanillaProxy.includes(`at Object.get`))
// Stealth tests
t.true(results.stealthProxy.includes(`TypeError: 'caller'`))
t.false(results.stealthProxy.includes(`at Object.get`))
})
test('replaceProperty: will work without traces', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await withUtils(page).evaluate(utils => {
utils.replaceProperty(Object.getPrototypeOf(navigator), 'languages', {
get: () => ['de-DE']
})
return {
propNames: Object.getOwnPropertyNames(navigator)
}
})
t.false(results.propNames.includes('languages'))
})
test('cache: will prevent leaks through overriding methods', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await withUtils(page).evaluate(utils => {
const sniffResults = {
vanilla: false,
stealth: false
}
const vanillaProxy = new Proxy(
{},
{
get() {
return Reflect.get(...arguments)
}
}
)
Reflect.get = () => (sniffResults.vanilla = true)
// trigger get trap
vanillaProxy.foo // eslint-disable-line
const stealthProxy = new Proxy(
{},
{
get() {
return utils.cache.Reflect.get(...arguments) // using cached copy
}
}
)
Reflect.get = () => (sniffResults.stealth = true)
// trigger get trap
stealthProxy.foo // eslint-disable-line
return sniffResults
})
t.deepEqual(results, {
vanilla: true,
stealth: false
})
})
test('replaceWithProxy: will throw prototype errors', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
await page.goto('about:blank')
const result = await withUtils(page).evaluate(utils => {
utils.replaceWithProxy(HTMLMediaElement.prototype, 'canPlayType', {})
const evalErr = (str = '') => {
try {
// eslint-disable-next-line no-eval
return eval(str)
} catch (err) {
return err.toString()
}
}
return {
same: evalErr(
`Object.setPrototypeOf(HTMLMediaElement.prototype.canPlayType, HTMLMediaElement.prototype.canPlayType) + ""`
),
sameString: evalErr(
`Object.setPrototypeOf(Function.prototype.toString, Function.prototype.toString) + ""`
),
null: evalErr(
`Object.setPrototypeOf(Function.prototype.toString, null) + ""`
),
undef: evalErr(
`Object.setPrototypeOf(Function.prototype.toString, undefined) + ""`
),
none: evalErr(`Object.setPrototypeOf(Function.prototype.toString) + ""`)
}
})
t.deepEqual(result, {
same: 'TypeError: Cyclic __proto__ value',
sameString: 'TypeError: Cyclic __proto__ value',
null: 'TypeError: Cannot convert object to primitive value',
undef:
'TypeError: Object prototype may only be an Object or null: undefined',
none: 'TypeError: Object prototype may only be an Object or null: undefined'
})
})
test('replaceGetterSetter', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
await page.goto('about:blank')
const results = await withUtils(page).evaluate(utils => {
const getDetails = a => ({
href: a.href,
typeof: typeof a.href,
in: 'href' in a,
keys: Object.keys(a),
// eslint-disable-next-line no-undef
prototypeKeys: Object.keys(HTMLAnchorElement.prototype),
getOwnPropertyNames: Object.getOwnPropertyNames(a),
prototypeGetOwnPropertyNames: Object.getOwnPropertyNames(
// eslint-disable-next-line no-undef
HTMLAnchorElement.prototype
),
ownPropertyDescriptor:
undefined === Object.getOwnPropertyDescriptor(a, 'href'),
prototypeOwnPropertyDescriptor: Object.getOwnPropertyDescriptor(
// eslint-disable-next-line no-undef
HTMLAnchorElement.prototype,
'href'
),
ownPropertyDescriptors: Object.getOwnPropertyDescriptors(a, 'href'),
prototypeOwnPropertyDescriptors: Object.getOwnPropertyDescriptors(
// eslint-disable-next-line no-undef
HTMLAnchorElement.prototype,
'href'
),
getToString: Object.getOwnPropertyDescriptor(
// eslint-disable-next-line no-undef
HTMLAnchorElement.prototype,
'href'
).get.toString(),
setToString: Object.getOwnPropertyDescriptor(
// eslint-disable-next-line no-undef
HTMLAnchorElement.prototype,
'href'
).set.toString()
})
// Use native a.href.
const a1 = document.createElement('a')
a1.href = 'http://foo.com/'
const details1 = getDetails(a1)
// Override a.href.
let href = ''
// eslint-disable-next-line no-undef
utils.replaceGetterSetter(HTMLAnchorElement.prototype, 'href', {
get: function() {
return href
},
set: function(newValue) {
href = newValue
}
})
// Use overrided a.href.
const a2 = document.createElement('a')
a2.href = 'http://foo.com/'
const details2 = getDetails(a2)
return [details1, details2]
})
t.deepEqual(results[1], results[0])
})
test('arrayEquals', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
await page.goto('about:blank')
const results = await withUtils(page).evaluate(utils => {
const obj = { foo: 'bar' }
return {
a: utils.arrayEquals(['a', 'Alpha'], ['a', 'Alpha']),
b: !utils.arrayEquals(['b', 'Beta'], ['b', 'Blue']),
c: !utils.arrayEquals(['c', { foo: 'bar' }], ['c', { foo: 'bar' }]),
d: utils.arrayEquals(['d', obj], ['d', obj]),
e: utils.arrayEquals([null], [null]),
f: utils.arrayEquals([undefined], [undefined]),
g: utils.arrayEquals([false], [false])
}
})
t.deepEqual(results, {
a: true,
b: true,
c: true,
d: true,
e: true,
f: true,
g: true
})
})
test('memoize', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
await page.goto('about:blank')
const results = await withUtils(page).evaluate(utils => {
const objectify = utils.memoize((valueAdded, valueIgnored) => {
return { valueAdded }
})
const obj = { foo: 'bar' }
/* eslint-disable no-self-compare */
return {
a: objectify('a', 'Alpha') === objectify('a', 'Alpha'),
b: objectify('b', 'Beta') !== objectify('b', 'Blue'),
c: objectify('c', { foo: 'bar' }) !== objectify('c', { foo: 'bar' }),
d: objectify('d', obj) === objectify('d', obj),
e: objectify(null) === objectify(null),
f: objectify(undefined) === objectify(undefined),
g: objectify(false) === objectify(false)
}
/* eslint-enable no-self-compare */
})
t.deepEqual(results, {
a: true,
b: true,
c: true,
d: true,
e: true,
f: true,
g: true
})
})

View File

@@ -0,0 +1,288 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [utils()](#utils)
- [.stripProxyFromErrors(handler)](#stripproxyfromerrorshandler)
- [.stripErrorWithAnchor(err, anchor)](#striperrorwithanchorerr-anchor)
- [.replaceProperty(obj, propName, descriptorOverrides)](#replacepropertyobj-propname-descriptoroverrides)
- [.preloadCache()](#preloadcache)
- [.makeNativeString(name?)](#makenativestringname)
- [.patchToString(obj, str)](#patchtostringobj-str)
- [.patchToStringNested(obj)](#patchtostringnestedobj)
- [.redirectToString(proxyObj, originalObj)](#redirecttostringproxyobj-originalobj)
- [.replaceWithProxy(obj, propName, handler)](#replacewithproxyobj-propname-handler)
- [.mockWithProxy(obj, propName, pseudoTarget, handler)](#mockwithproxyobj-propname-pseudotarget-handler)
- [.createProxy(pseudoTarget, handler)](#createproxypseudotarget-handler)
- [.splitObjPath(objPath)](#splitobjpathobjpath)
- [.replaceObjPathWithProxy(objPath, handler)](#replaceobjpathwithproxyobjpath-handler)
- [.execRecursively(obj, typeFilter, fn)](#execrecursivelyobj-typefilter-fn)
- [.stringifyFns(fnObj)](#stringifyfnsfnobj)
- [.materializeFns(fnStrObj)](#materializefnsfnstrobj)
### [utils()](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L12-L12)
A set of shared utility functions specifically for the purpose of modifying native browser APIs without leaving traces.
Meant to be passed down in puppeteer and used in the context of the page (everything in here runs in NodeJS as well as a browser).
Note: If for whatever reason you need to use this outside of `puppeteer-extra`:
Just remove the `module.exports` statement at the very bottom, the rest can be copy pasted into any browser context.
Alternatively take a look at the `extract-stealth-evasions` package to create a finished bundle which includes these utilities.
---
#### .[stripProxyFromErrors(handler)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L21-L82)
- `handler` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** The JS Proxy handler to wrap (optional, default `{}`)
Wraps a JS Proxy Handler and strips it's presence from error stacks, in case the traps throw.
The presence of a JS Proxy can be revealed as it shows up in error stack traces.
---
#### .[stripErrorWithAnchor(err, anchor)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L90-L101)
- `err` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** The error to sanitize
- `anchor` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The string the anchor line starts with
Strip error lines from stack traces until (and including) a known line the stack.
---
#### .[replaceProperty(obj, propName, descriptorOverrides)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L120-L127)
- `obj` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** The object which has the property to replace
- `propName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The property name to replace
- `descriptorOverrides` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** e.g. { value: "alice" } (optional, default `{}`)
Replace the property of an object in a stealthy way.
Note: You also want to work on the prototype of an object most often,
as you'd otherwise leave traces (e.g. showing up in Object.getOwnPropertyNames(obj)).
Example:
```javascript
replaceProperty(WebGLRenderingContext.prototype, 'getParameter', {
value: 'alice'
})
// or
replaceProperty(Object.getPrototypeOf(navigator), 'languages', {
get: () => ['en-US', 'en']
})
```
- **See: <https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty>**
---
#### .[preloadCache()](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L137-L150)
Preload a cache of function copies and data.
For a determined enough observer it would be possible to overwrite and sniff usage of functions
we use in our internal Proxies, to combat that we use a cached copy of those functions.
This is evaluated once per execution context (e.g. window)
---
#### .[makeNativeString(name?)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L169-L173)
- `name` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** Optional function name (optional, default `''`)
Utility function to generate a cross-browser `toString` result representing native code.
There's small differences: Chromium uses a single line, whereas FF & Webkit uses multiline strings.
To future-proof this we use an existing native toString result as the basis.
The only advantage we have over the other team is that our JS runs first, hence we cache the result
of the native toString result once, so they cannot spoof it afterwards and reveal that we're using it.
Note: Whenever we add a `Function.prototype.toString` proxy we should preload the cache before,
by executing `utils.preloadCache()` before the proxy is applied (so we don't cause recursive lookups).
Example:
```javascript
makeNativeString('foobar') // => `function foobar() { [native code] }`
```
---
#### .[patchToString(obj, str)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L189-L218)
- `obj` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** The object for which to modify the `toString()` representation
- `str` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Optional string used as a return value (optional, default `''`)
Helper function to modify the `toString()` result of the provided object.
Note: Use `utils.redirectToString` instead when possible.
There's a quirk in JS Proxies that will cause the `toString()` result to differ from the vanilla Object.
If no string is provided we will generate a `[native code]` thing based on the name of the property object.
Example:
```javascript
patchToString(
WebGLRenderingContext.prototype.getParameter,
'function getParameter() { [native code] }'
)
```
---
#### .[patchToStringNested(obj)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L225-L227)
- `obj` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** (optional, default `{}`)
Make all nested functions of an object native.
---
#### .[redirectToString(proxyObj, originalObj)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L235-L272)
- `proxyObj` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** The object that toString will be called on
- `originalObj` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** The object which toString result we wan to return
Redirect toString requests from one object to another.
---
#### .[replaceWithProxy(obj, propName, handler)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L287-L296)
- `obj` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** The object which has the property to replace
- `propName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The name of the property to replace
- `handler` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** The JS Proxy handler to use
All-in-one method to replace a property with a JS Proxy using the provided Proxy handler with traps.
Will stealthify these aspects (strip error stack traces, redirect toString, etc).
Note: This is meant to modify native Browser APIs and works best with prototype objects.
Example:
```javascript
replaceWithProxy(WebGLRenderingContext.prototype, 'getParameter', proxyHandler)
```
---
#### .[mockWithProxy(obj, propName, pseudoTarget, handler)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L311-L319)
- `obj` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** The object which has the property to replace
- `propName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The name of the property to replace or create
- `pseudoTarget` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** The JS Proxy target to use as a basis
- `handler` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** The JS Proxy handler to use
All-in-one method to mock a non-existing property with a JS Proxy using the provided Proxy handler with traps.
Will stealthify these aspects (strip error stack traces, redirect toString, etc).
Example:
```javascript
mockWithProxy(
chrome.runtime,
'sendMessage',
function sendMessage() {},
proxyHandler
)
```
---
#### .[createProxy(pseudoTarget, handler)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L334-L340)
- `pseudoTarget` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** The JS Proxy target to use as a basis
- `handler` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** The JS Proxy handler to use
All-in-one method to create a new JS Proxy with stealth tweaks.
This is meant to be used whenever we need a JS Proxy but don't want to replace or mock an existing known property.
Will stealthify certain aspects of the Proxy (strip error stack traces, redirect toString, etc).
Example:
```javascript
createProxy(navigator.mimeTypes.__proto__.namedItem, proxyHandler) // => Proxy
```
---
#### .[splitObjPath(objPath)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L351-L359)
- `objPath` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The full path to an object as dot notation string
Helper function to split a full path to an Object into the first part and property.
Example:
```javascript
splitObjPath(`HTMLMediaElement.prototype.canPlayType`)
// => {objName: "HTMLMediaElement.prototype", propName: "canPlayType"}
```
---
#### .[replaceObjPathWithProxy(objPath, handler)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L372-L376)
- `objPath` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The full path to an object (dot notation string) to replace
- `handler` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** The JS Proxy handler to use
Convenience method to replace a property with a JS Proxy using the provided objPath.
Supports a full path (dot notation) to the object as string here, in case that makes it easier.
Example:
```javascript
replaceObjPathWithProxy(
'WebGLRenderingContext.prototype.getParameter',
proxyHandler
)
```
---
#### .[execRecursively(obj, typeFilter, fn)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L385-L402)
- `obj` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** (optional, default `{}`)
- `typeFilter` **[array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** e.g. `['function']` (optional, default `[]`)
- `fn` **[Function](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function)** e.g. `utils.patchToString`
Traverse nested properties of an object recursively and apply the given function on a whitelist of value types.
---
#### .[stringifyFns(fnObj)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L417-L431)
- `fnObj` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** An object containing functions as properties (optional, default `{hello:()=>'world'}`)
Everything we run through e.g. `page.evaluate` runs in the browser context, not the NodeJS one.
That means we cannot just use reference variables and functions from outside code, we need to pass everything as a parameter.
Unfortunately the data we can pass is only allowed to be of primitive types, regular functions don't survive the built-in serialization process.
This utility function will take an object with functions and stringify them, so we can pass them down unharmed as strings.
We use this to pass down our utility functions as well as any other functions (to be able to split up code better).
- **See: utils.materializeFns**
---
#### .[materializeFns(fnStrObj)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js#L439-L451)
- `fnStrObj` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** An object containing stringified functions as properties (optional, default `{hello:"() => 'world'"}`)
Utility function to reverse the process of `utils.stringifyFns`.
Will materialize an object with stringified functions (supports classic and fat arrow functions).
---

View File

@@ -0,0 +1,49 @@
const utils = require('./index')
/**
* Wrap a page with utilities.
*
* @param {Puppeteer.Page} page
*/
module.exports = page => ({
/**
* Simple `page.evaluate` replacement to preload utils
*/
evaluate: async function (mainFunction, ...args) {
return page.evaluate(
({ _utilsFns, _mainFunction, _args }) => {
// Add this point we cannot use our utililty functions as they're just strings, we need to materialize them first
const utils = Object.fromEntries(
Object.entries(_utilsFns).map(([key, value]) => [key, eval(value)]) // eslint-disable-line no-eval
)
utils.init()
return eval(_mainFunction)(utils, ..._args) // eslint-disable-line no-eval
},
{
_utilsFns: utils.stringifyFns(utils),
_mainFunction: mainFunction.toString(),
_args: args || []
}
)
},
/**
* Simple `page.evaluateOnNewDocument` replacement to preload utils
*/
evaluateOnNewDocument: async function (mainFunction, ...args) {
return page.evaluateOnNewDocument(
({ _utilsFns, _mainFunction, _args }) => {
// Add this point we cannot use our utililty functions as they're just strings, we need to materialize them first
const utils = Object.fromEntries(
Object.entries(_utilsFns).map(([key, value]) => [key, eval(value)]) // eslint-disable-line no-eval
)
utils.init()
return eval(_mainFunction)(utils, ..._args) // eslint-disable-line no-eval
},
{
_utilsFns: utils.stringifyFns(utils),
_mainFunction: mainFunction.toString(),
_args: args || []
}
)
}
})

View File

@@ -0,0 +1,100 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const withUtils = require('../_utils/withUtils')
/**
* Mock the `chrome.app` object if not available (e.g. when running headless).
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/chrome.app'
}
async onPageCreated(page) {
await withUtils(page).evaluateOnNewDocument(utils => {
if (!window.chrome) {
// Use the exact property descriptor found in headful Chrome
// fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')`
Object.defineProperty(window, 'chrome', {
writable: true,
enumerable: true,
configurable: false, // note!
value: {} // We'll extend that later
})
}
// That means we're running headful and don't need to mock anything
if ('app' in window.chrome) {
return // Nothing to do here
}
const makeError = {
ErrorInInvocation: fn => {
const err = new TypeError(`Error in invocation of app.${fn}()`)
return utils.stripErrorWithAnchor(
err,
`at ${fn} (eval at <anonymous>`
)
}
}
// There's a some static data in that property which doesn't seem to change,
// we should periodically check for updates: `JSON.stringify(window.app, null, 2)`
const STATIC_DATA = JSON.parse(
`
{
"isInstalled": false,
"InstallState": {
"DISABLED": "disabled",
"INSTALLED": "installed",
"NOT_INSTALLED": "not_installed"
},
"RunningState": {
"CANNOT_RUN": "cannot_run",
"READY_TO_RUN": "ready_to_run",
"RUNNING": "running"
}
}
`.trim()
)
window.chrome.app = {
...STATIC_DATA,
get isInstalled() {
return false
},
getDetails: function getDetails() {
if (arguments.length) {
throw makeError.ErrorInInvocation(`getDetails`)
}
return null
},
getIsInstalled: function getDetails() {
if (arguments.length) {
throw makeError.ErrorInInvocation(`getIsInstalled`)
}
return false
},
runningState: function getDetails() {
if (arguments.length) {
throw makeError.ErrorInInvocation(`runningState`)
}
return 'cannot_run'
}
}
utils.patchToStringNested(window.chrome.app)
})
}
}
module.exports = function(pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,71 @@
const test = require('ava')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
/* global chrome */
test('stealth: will add convincing chrome.app object', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin({}))
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await page.evaluate(() => {
const catchErr = (fn, ...args) => {
try {
return fn.apply(this, args)
} catch ({ name, message, stack }) {
return { name, message, stack }
}
}
return {
app: {
exists: window.chrome && 'app' in window.chrome,
toString: chrome.app.toString(),
deepToString: chrome.app.runningState.toString()
},
data: {
getIsInstalled: chrome.app.getIsInstalled(),
runningState: chrome.app.runningState(),
getDetails: chrome.app.getDetails(),
InstallState: chrome.app.InstallState,
RunningState: chrome.app.RunningState
},
errors: {
getIsInstalled: catchErr(chrome.app.getDetails, 'foo').message,
stackOK: !catchErr(chrome.app.getDetails, 'foo').stack.includes(
'at getDetails'
)
}
}
})
t.deepEqual(results, {
app: {
exists: true,
toString: '[object Object]',
deepToString: 'function getDetails() { [native code] }'
},
data: {
InstallState: {
DISABLED: 'disabled',
INSTALLED: 'installed',
NOT_INSTALLED: 'not_installed'
},
RunningState: {
CANNOT_RUN: 'cannot_run',
READY_TO_RUN: 'ready_to_run',
RUNNING: 'running'
},
getDetails: null,
getIsInstalled: false,
runningState: 'cannot_run'
},
errors: {
getIsInstalled: 'Error in invocation of app.getDetails()',
stackOK: true
}
})
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,17 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/chrome.app/index.js#L11-L97)
- `opts` (optional, default `{}`)
**Extends: PuppeteerExtraPlugin**
Mock the `chrome.app` object if not available (e.g. when running headless).
---

View File

@@ -0,0 +1,73 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const withUtils = require('../_utils/withUtils')
/**
* Mock the `chrome.csi` function if not available (e.g. when running headless).
* It's a deprecated (but unfortunately still existing) chrome specific API to fetch browser timings.
*
* Internally chromium switched the implementation to use the WebPerformance API,
* so we can do the same to create a fully functional mock. :-)
*
* Note: We're using the deprecated PerformanceTiming API instead of the new Navigation Timing Level 2 API on purpopse.
*
* @see https://bugs.chromium.org/p/chromium/issues/detail?id=113048
* @see https://codereview.chromium.org/2456293003/
* @see https://developers.google.com/web/updates/2017/12/chrome-loadtimes-deprecated
* @see https://developer.mozilla.org/en-US/docs/Web/API/PerformanceTiming
* @see https://source.chromium.org/chromium/chromium/src/+/master:chrome/renderer/loadtimes_extension_bindings.cc;l=124?q=loadtimes&ss=chromium
* @see `chrome.loadTimes` evasion
*
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/chrome.csi'
}
async onPageCreated(page) {
await withUtils(page).evaluateOnNewDocument(utils => {
if (!window.chrome) {
// Use the exact property descriptor found in headful Chrome
// fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')`
Object.defineProperty(window, 'chrome', {
writable: true,
enumerable: true,
configurable: false, // note!
value: {} // We'll extend that later
})
}
// That means we're running headful and don't need to mock anything
if ('csi' in window.chrome) {
return // Nothing to do here
}
// Check that the Navigation Timing API v1 is available, we need that
if (!window.performance || !window.performance.timing) {
return
}
const { timing } = window.performance
window.chrome.csi = function() {
return {
onloadT: timing.domContentLoadedEventEnd,
startE: timing.navigationStart,
pageT: Date.now() - timing.navigationStart,
tran: 15 // Transition type or something
}
}
utils.patchToString(window.chrome.csi)
})
}
}
module.exports = function(pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,48 @@
const test = require('ava')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
/* global chrome */
test('stealth: will add functional chrome.csi function mock', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(
Plugin({
runOnInsecureOrigins: true // for testing
})
)
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await page.evaluate(() => {
const { timing } = window.performance
const csi = window.chrome.csi()
return {
csi: {
exists: window.chrome && 'csi' in window.chrome,
toString: chrome.csi.toString()
},
dataOK: {
onloadT: csi.onloadT === timing.domContentLoadedEventEnd,
startE: csi.startE === timing.navigationStart,
pageT: Number.isInteger(csi.pageT),
tran: Number.isInteger(csi.tran)
}
}
})
t.deepEqual(results, {
csi: {
exists: true,
toString: 'function () { [native code] }'
},
dataOK: {
onloadT: true,
pageT: true,
startE: true,
tran: true
}
})
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,30 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/chrome.csi/index.js#L25-L70)
- `opts` (optional, default `{}`)
**Extends: PuppeteerExtraPlugin**
Mock the `chrome.csi` function if not available (e.g. when running headless).
It's a deprecated (but unfortunately still existing) chrome specific API to fetch browser timings.
Internally chromium switched the implementation to use the WebPerformance API,
so we can do the same to create a fully functional mock. :-)
Note: We're using the deprecated PerformanceTiming API instead of the new Navigation Timing Level 2 API on purpopse.
- **See: <https://bugs.chromium.org/p/chromium/issues/detail?id=113048>**
- **See: <https://codereview.chromium.org/2456293003/>**
- **See: <https://developers.google.com/web/updates/2017/12/chrome-loadtimes-deprecated>**
- **See: <https://developer.mozilla.org/en-US/docs/Web/API/PerformanceTiming>**
- **See: <https://source.chromium.org/chromium/chromium/src/+/master:chrome/renderer/loadtimes_extension_bindings.cc;l=124?q=loadtimes&ss=chromium>**
- **See: `chrome.loadTimes` evasion**
---

View File

@@ -0,0 +1,167 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const withUtils = require('../_utils/withUtils')
/**
* Mock the `chrome.loadTimes` function if not available (e.g. when running headless).
* It's a deprecated (but unfortunately still existing) chrome specific API to fetch browser timings and connection info.
*
* Internally chromium switched the implementation to use the WebPerformance API,
* so we can do the same to create a fully functional mock. :-)
*
* Note: We're using the deprecated PerformanceTiming API instead of the new Navigation Timing Level 2 API on purpopse.
*
* @see https://developers.google.com/web/updates/2017/12/chrome-loadtimes-deprecated
* @see https://developer.mozilla.org/en-US/docs/Web/API/PerformanceTiming
* @see https://source.chromium.org/chromium/chromium/src/+/master:chrome/renderer/loadtimes_extension_bindings.cc;l=124?q=loadtimes&ss=chromium
* @see `chrome.csi` evasion
*
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/chrome.loadTimes'
}
async onPageCreated(page) {
await withUtils(page).evaluateOnNewDocument(
(utils, { opts }) => {
if (!window.chrome) {
// Use the exact property descriptor found in headful Chrome
// fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')`
Object.defineProperty(window, 'chrome', {
writable: true,
enumerable: true,
configurable: false, // note!
value: {} // We'll extend that later
})
}
// That means we're running headful and don't need to mock anything
if ('loadTimes' in window.chrome) {
return // Nothing to do here
}
// Check that the Navigation Timing API v1 + v2 is available, we need that
if (
!window.performance ||
!window.performance.timing ||
!window.PerformancePaintTiming
) {
return
}
const { performance } = window
// Some stuff is not available on about:blank as it requires a navigation to occur,
// let's harden the code to not fail then:
const ntEntryFallback = {
nextHopProtocol: 'h2',
type: 'other'
}
// The API exposes some funky info regarding the connection
const protocolInfo = {
get connectionInfo() {
const ntEntry =
performance.getEntriesByType('navigation')[0] || ntEntryFallback
return ntEntry.nextHopProtocol
},
get npnNegotiatedProtocol() {
// NPN is deprecated in favor of ALPN, but this implementation returns the
// HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN.
const ntEntry =
performance.getEntriesByType('navigation')[0] || ntEntryFallback
return ['h2', 'hq'].includes(ntEntry.nextHopProtocol)
? ntEntry.nextHopProtocol
: 'unknown'
},
get navigationType() {
const ntEntry =
performance.getEntriesByType('navigation')[0] || ntEntryFallback
return ntEntry.type
},
get wasAlternateProtocolAvailable() {
// The Alternate-Protocol header is deprecated in favor of Alt-Svc
// (https://www.mnot.net/blog/2016/03/09/alt-svc), so technically this
// should always return false.
return false
},
get wasFetchedViaSpdy() {
// SPDY is deprecated in favor of HTTP/2, but this implementation returns
// true for HTTP/2 or HTTP2+QUIC/39 as well.
const ntEntry =
performance.getEntriesByType('navigation')[0] || ntEntryFallback
return ['h2', 'hq'].includes(ntEntry.nextHopProtocol)
},
get wasNpnNegotiated() {
// NPN is deprecated in favor of ALPN, but this implementation returns true
// for HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN.
const ntEntry =
performance.getEntriesByType('navigation')[0] || ntEntryFallback
return ['h2', 'hq'].includes(ntEntry.nextHopProtocol)
}
}
const { timing } = window.performance
// Truncate number to specific number of decimals, most of the `loadTimes` stuff has 3
function toFixed(num, fixed) {
var re = new RegExp('^-?\\d+(?:.\\d{0,' + (fixed || -1) + '})?')
return num.toString().match(re)[0]
}
const timingInfo = {
get firstPaintAfterLoadTime() {
// This was never actually implemented and always returns 0.
return 0
},
get requestTime() {
return timing.navigationStart / 1000
},
get startLoadTime() {
return timing.navigationStart / 1000
},
get commitLoadTime() {
return timing.responseStart / 1000
},
get finishDocumentLoadTime() {
return timing.domContentLoadedEventEnd / 1000
},
get finishLoadTime() {
return timing.loadEventEnd / 1000
},
get firstPaintTime() {
const fpEntry = performance.getEntriesByType('paint')[0] || {
startTime: timing.loadEventEnd / 1000 // Fallback if no navigation occured (`about:blank`)
}
return toFixed(
(fpEntry.startTime + performance.timeOrigin) / 1000,
3
)
}
}
window.chrome.loadTimes = function() {
return {
...protocolInfo,
...timingInfo
}
}
utils.patchToString(window.chrome.loadTimes)
},
{
opts: this.opts
}
)
}
}
module.exports = function(pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,63 @@
const test = require('ava')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
/* global chrome */
test('stealth: will add functional chrome.loadTimes function mock', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin({}))
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await page.evaluate(() => {
const loadTimes = window.chrome.loadTimes()
return {
loadTimes: {
exists: window.chrome && 'loadTimes' in window.chrome,
toString: chrome.loadTimes.toString()
},
dataOK: {
connectionInfo: 'connectionInfo' in loadTimes,
npnNegotiatedProtocol: 'npnNegotiatedProtocol' in loadTimes,
navigationType: 'navigationType' in loadTimes,
wasAlternateProtocolAvailable:
'wasAlternateProtocolAvailable' in loadTimes,
wasFetchedViaSpdy: 'wasFetchedViaSpdy' in loadTimes,
wasNpnNegotiated: 'wasNpnNegotiated' in loadTimes,
firstPaintAfterLoadTime: 'firstPaintAfterLoadTime' in loadTimes,
requestTime: 'requestTime' in loadTimes,
startLoadTime: 'startLoadTime' in loadTimes,
commitLoadTime: 'commitLoadTime' in loadTimes,
finishDocumentLoadTime: 'finishDocumentLoadTime' in loadTimes,
finishLoadTime: 'finishLoadTime' in loadTimes,
firstPaintTime: 'firstPaintTime' in loadTimes
}
}
})
t.deepEqual(results, {
loadTimes: {
exists: true,
toString: 'function () { [native code] }'
},
dataOK: {
commitLoadTime: true,
connectionInfo: true,
finishDocumentLoadTime: true,
finishLoadTime: true,
firstPaintAfterLoadTime: true,
firstPaintTime: true,
navigationType: true,
npnNegotiatedProtocol: true,
requestTime: true,
startLoadTime: true,
wasAlternateProtocolAvailable: true,
wasFetchedViaSpdy: true,
wasNpnNegotiated: true
}
})
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,28 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes/index.js#L23-L164)
- `opts` (optional, default `{}`)
**Extends: PuppeteerExtraPlugin**
Mock the `chrome.loadTimes` function if not available (e.g. when running headless).
It's a deprecated (but unfortunately still existing) chrome specific API to fetch browser timings and connection info.
Internally chromium switched the implementation to use the WebPerformance API,
so we can do the same to create a fully functional mock. :-)
Note: We're using the deprecated PerformanceTiming API instead of the new Navigation Timing Level 2 API on purpopse.
- **See: <https://developers.google.com/web/updates/2017/12/chrome-loadtimes-deprecated>**
- **See: <https://developer.mozilla.org/en-US/docs/Web/API/PerformanceTiming>**
- **See: <https://source.chromium.org/chromium/chromium/src/+/master:chrome/renderer/loadtimes_extension_bindings.cc;l=124?q=loadtimes&ss=chromium>**
- **See: `chrome.csi` evasion**
---

View File

@@ -0,0 +1,254 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const withUtils = require('../_utils/withUtils')
const STATIC_DATA = require('./staticData.json')
/**
* Mock the `chrome.runtime` object if not available (e.g. when running headless) and on a secure site.
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/chrome.runtime'
}
get defaults() {
return { runOnInsecureOrigins: false } // Override for testing
}
async onPageCreated(page) {
await withUtils(page).evaluateOnNewDocument(
(utils, { opts, STATIC_DATA }) => {
if (!window.chrome) {
// Use the exact property descriptor found in headful Chrome
// fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')`
Object.defineProperty(window, 'chrome', {
writable: true,
enumerable: true,
configurable: false, // note!
value: {} // We'll extend that later
})
}
// That means we're running headful and don't need to mock anything
const existsAlready = 'runtime' in window.chrome
// `chrome.runtime` is only exposed on secure origins
const isNotSecure = !window.location.protocol.startsWith('https')
if (existsAlready || (isNotSecure && !opts.runOnInsecureOrigins)) {
return // Nothing to do here
}
window.chrome.runtime = {
// There's a bunch of static data in that property which doesn't seem to change,
// we should periodically check for updates: `JSON.stringify(window.chrome.runtime, null, 2)`
...STATIC_DATA,
// `chrome.runtime.id` is extension related and returns undefined in Chrome
get id() {
return undefined
},
// These two require more sophisticated mocks
connect: null,
sendMessage: null
}
const makeCustomRuntimeErrors = (preamble, method, extensionId) => ({
NoMatchingSignature: new TypeError(
preamble + `No matching signature.`
),
MustSpecifyExtensionID: new TypeError(
preamble +
`${method} called from a webpage must specify an Extension ID (string) for its first argument.`
),
InvalidExtensionID: new TypeError(
preamble + `Invalid extension id: '${extensionId}'`
)
})
// Valid Extension IDs are 32 characters in length and use the letter `a` to `p`:
// https://source.chromium.org/chromium/chromium/src/+/master:components/crx_file/id_util.cc;drc=14a055ccb17e8c8d5d437fe080faba4c6f07beac;l=90
const isValidExtensionID = str =>
str.length === 32 && str.toLowerCase().match(/^[a-p]+$/)
/** Mock `chrome.runtime.sendMessage` */
const sendMessageHandler = {
apply: function(target, ctx, args) {
const [extensionId, options, responseCallback] = args || []
// Define custom errors
const errorPreamble = `Error in invocation of runtime.sendMessage(optional string extensionId, any message, optional object options, optional function responseCallback): `
const Errors = makeCustomRuntimeErrors(
errorPreamble,
`chrome.runtime.sendMessage()`,
extensionId
)
// Check if the call signature looks ok
const noArguments = args.length === 0
const tooManyArguments = args.length > 4
const incorrectOptions = options && typeof options !== 'object'
const incorrectResponseCallback =
responseCallback && typeof responseCallback !== 'function'
if (
noArguments ||
tooManyArguments ||
incorrectOptions ||
incorrectResponseCallback
) {
throw Errors.NoMatchingSignature
}
// At least 2 arguments are required before we even validate the extension ID
if (args.length < 2) {
throw Errors.MustSpecifyExtensionID
}
// Now let's make sure we got a string as extension ID
if (typeof extensionId !== 'string') {
throw Errors.NoMatchingSignature
}
if (!isValidExtensionID(extensionId)) {
throw Errors.InvalidExtensionID
}
return undefined // Normal behavior
}
}
utils.mockWithProxy(
window.chrome.runtime,
'sendMessage',
function sendMessage() {},
sendMessageHandler
)
/**
* Mock `chrome.runtime.connect`
*
* @see https://developer.chrome.com/apps/runtime#method-connect
*/
const connectHandler = {
apply: function(target, ctx, args) {
const [extensionId, connectInfo] = args || []
// Define custom errors
const errorPreamble = `Error in invocation of runtime.connect(optional string extensionId, optional object connectInfo): `
const Errors = makeCustomRuntimeErrors(
errorPreamble,
`chrome.runtime.connect()`,
extensionId
)
// Behavior differs a bit from sendMessage:
const noArguments = args.length === 0
const emptyStringArgument = args.length === 1 && extensionId === ''
if (noArguments || emptyStringArgument) {
throw Errors.MustSpecifyExtensionID
}
const tooManyArguments = args.length > 2
const incorrectConnectInfoType =
connectInfo && typeof connectInfo !== 'object'
if (tooManyArguments || incorrectConnectInfoType) {
throw Errors.NoMatchingSignature
}
const extensionIdIsString = typeof extensionId === 'string'
if (extensionIdIsString && extensionId === '') {
throw Errors.MustSpecifyExtensionID
}
if (extensionIdIsString && !isValidExtensionID(extensionId)) {
throw Errors.InvalidExtensionID
}
// There's another edge-case here: extensionId is optional so we might find a connectInfo object as first param, which we need to validate
const validateConnectInfo = ci => {
// More than a first param connectInfo as been provided
if (args.length > 1) {
throw Errors.NoMatchingSignature
}
// An empty connectInfo has been provided
if (Object.keys(ci).length === 0) {
throw Errors.MustSpecifyExtensionID
}
// Loop over all connectInfo props an check them
Object.entries(ci).forEach(([k, v]) => {
const isExpected = ['name', 'includeTlsChannelId'].includes(k)
if (!isExpected) {
throw new TypeError(
errorPreamble + `Unexpected property: '${k}'.`
)
}
const MismatchError = (propName, expected, found) =>
TypeError(
errorPreamble +
`Error at property '${propName}': Invalid type: expected ${expected}, found ${found}.`
)
if (k === 'name' && typeof v !== 'string') {
throw MismatchError(k, 'string', typeof v)
}
if (k === 'includeTlsChannelId' && typeof v !== 'boolean') {
throw MismatchError(k, 'boolean', typeof v)
}
})
}
if (typeof extensionId === 'object') {
validateConnectInfo(extensionId)
throw Errors.MustSpecifyExtensionID
}
// Unfortunately even when the connect fails Chrome will return an object with methods we need to mock as well
return utils.patchToStringNested(makeConnectResponse())
}
}
utils.mockWithProxy(
window.chrome.runtime,
'connect',
function connect() {},
connectHandler
)
function makeConnectResponse() {
const onSomething = () => ({
addListener: function addListener() {},
dispatch: function dispatch() {},
hasListener: function hasListener() {},
hasListeners: function hasListeners() {
return false
},
removeListener: function removeListener() {}
})
const response = {
name: '',
sender: undefined,
disconnect: function disconnect() {},
onDisconnect: onSomething(),
onMessage: onSomething(),
postMessage: function postMessage() {
if (!arguments.length) {
throw new TypeError(`Insufficient number of arguments.`)
}
throw new Error(`Attempting to use a disconnected port object`)
}
}
return response
}
},
{
opts: this.opts,
STATIC_DATA
}
)
}
}
module.exports = function(pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,286 @@
const test = require('ava')
const {
getVanillaFingerPrint,
getStealthFingerPrint
} = require('../../test/util')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
const STATIC_DATA = require('./staticData.json')
/* global chrome */
test('vanilla: is chrome false', async t => {
const pageFn = async page => await page.evaluate(() => window.chrome) // eslint-disable-line
const { pageFnResult: chrome, hasChrome } = await getVanillaFingerPrint(
pageFn
)
t.is(hasChrome, false)
t.false(chrome instanceof Object)
t.is(chrome, undefined)
})
test('stealth: is chrome true', async t => {
const pageFn = async page => await page.evaluate(() => window.chrome) // eslint-disable-line
const { pageFnResult: chrome, hasChrome } = await getStealthFingerPrint(
Plugin,
pageFn
)
t.is(hasChrome, true)
t.true(chrome instanceof Object)
})
test('stealth: will add convincing chrome.runtime object', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(
Plugin({
runOnInsecureOrigins: true // for testing
})
)
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
//
const results = await page.evaluate(() => {
const catchErr = (fn, ...args) => {
try {
return fn.apply(this, args)
} catch (err) {
return err.toString()
}
}
return {
runtime: {
exists: window.chrome && 'runtime' in window.chrome,
toString: chrome.runtime.toString()
},
staticData: {
OnInstalledReason: chrome.runtime.OnInstalledReason,
OnRestartRequiredReason: chrome.runtime.OnRestartRequiredReason,
PlatformArch: chrome.runtime.PlatformArch,
PlatformNaclArch: chrome.runtime.PlatformNaclArch,
PlatformOs: chrome.runtime.PlatformOs,
RequestUpdateCheckStatus: chrome.runtime.RequestUpdateCheckStatus
},
id: {
exists: 'id' in chrome.runtime,
undefined: chrome.runtime.id === undefined
},
sendMessage: {
exists: 'sendMessage' in chrome.runtime,
name: chrome.runtime.sendMessage.name,
toString1: chrome.runtime.sendMessage + '',
toString2: chrome.runtime.sendMessage.toString(),
validIdWorks:
chrome.runtime.sendMessage('nckgahadagoaajjgafhacjanaoiihapd', '') ===
undefined
},
sendMessageErrors: {
noArg: catchErr(chrome.runtime.sendMessage),
singleArg: catchErr(chrome.runtime.sendMessage, ''),
tooManyArg: catchErr(
chrome.runtime.sendMessage,
'',
'',
'',
'',
'',
''
),
incorrectArg: catchErr(chrome.runtime.sendMessage, '', '', {}, ''),
noValidID: catchErr(chrome.runtime.sendMessage, 'foo', '')
}
}
})
const bla = `TypeError: Error in invocation of runtime.sendMessage(optional string extensionId, any message, optional object options, optional function responseCallback)`
t.deepEqual(results, {
runtime: {
exists: true,
toString: '[object Object]'
},
staticData: STATIC_DATA,
id: {
exists: true,
undefined: true
},
sendMessage: {
exists: true,
name: 'sendMessage',
toString1: 'function sendMessage() { [native code] }',
toString2: 'function sendMessage() { [native code] }',
validIdWorks: true
},
sendMessageErrors: {
noArg: `${bla}: No matching signature.`,
singleArg: `${bla}: chrome.runtime.sendMessage() called from a webpage must specify an Extension ID (string) for its first argument.`,
tooManyArg: `${bla}: No matching signature.`,
incorrectArg: `${bla}: No matching signature.`,
noValidID: `${bla}: Invalid extension id: 'foo'`
}
})
})
test('stealth: will add convincing chrome.runtime.connect', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(
Plugin({
runOnInsecureOrigins: true // for testing
})
)
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await page.evaluate(() => {
const catchErr = (fn, ...args) => {
try {
return fn.apply(this, args)
} catch (err) {
return err.toString()
}
}
return {
connect: {
exists: 'connect' in chrome.runtime,
name: chrome.runtime.connect.name,
toString1: chrome.runtime.connect + '',
toString2: chrome.runtime.connect.toString(),
validIdWorks:
chrome.runtime.connect('nckgahadagoaajjgafhacjanaoiihapd') !==
undefined
},
connectErrors: {
noArg: catchErr(chrome.runtime.connect),
singleArg: catchErr(chrome.runtime.connect, ''),
tooManyArg: catchErr(chrome.runtime.connect, '', '', '', '', '', ''),
incorrectArg: catchErr(chrome.runtime.connect, '', '', {}, ''),
noValidID: catchErr(chrome.runtime.connect, 'foo', ''),
connectInfoFirst: {
emptyObject: catchErr(chrome.runtime.connect, {}),
tooManyArg: catchErr(chrome.runtime.connect, {}, {}),
unexpectedProp: catchErr(chrome.runtime.connect, { wtf: true }),
invalidName: catchErr(chrome.runtime.connect, { name: 666 }),
invalidTLS: catchErr(chrome.runtime.connect, {
includeTlsChannelId: 777
}),
invalidBoth: catchErr(chrome.runtime.connect, {
name: 666,
includeTlsChannelId: 777
}),
validName: catchErr(chrome.runtime.connect, { name: 'foo' }),
missingExtensionId: catchErr(chrome.runtime.connect, {
name: 'bob',
includeTlsChannelId: false
})
}
}
}
})
const bla = `TypeError: Error in invocation of runtime.connect(optional string extensionId, optional object connectInfo)`
t.deepEqual(results, {
connect: {
exists: true,
name: 'connect',
toString1: 'function connect() { [native code] }',
toString2: 'function connect() { [native code] }',
validIdWorks: true
},
connectErrors: {
noArg: `${bla}: chrome.runtime.connect() called from a webpage must specify an Extension ID (string) for its first argument.`,
singleArg: `${bla}: chrome.runtime.connect() called from a webpage must specify an Extension ID (string) for its first argument.`,
tooManyArg: `${bla}: No matching signature.`,
incorrectArg: `${bla}: No matching signature.`,
noValidID: `${bla}: Invalid extension id: 'foo'`,
connectInfoFirst: {
emptyObject: `${bla}: chrome.runtime.connect() called from a webpage must specify an Extension ID (string) for its first argument.`,
tooManyArg: `${bla}: No matching signature.`,
unexpectedProp: `${bla}: Unexpected property: 'wtf'.`,
invalidName: `${bla}: Error at property 'name': Invalid type: expected string, found number.`,
invalidTLS: `${bla}: Error at property 'includeTlsChannelId': Invalid type: expected boolean, found number.`,
invalidBoth: `${bla}: Error at property 'name': Invalid type: expected string, found number.`,
validName: `${bla}: chrome.runtime.connect() called from a webpage must specify an Extension ID (string) for its first argument.`,
missingExtensionId: `${bla}: chrome.runtime.connect() called from a webpage must specify an Extension ID (string) for its first argument.`
}
}
})
})
test('stealth: will add convincing chrome.runtime.connect response', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(
Plugin({
runOnInsecureOrigins: true // for testing
})
)
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await page.evaluate(() => {
const connectResponse = chrome.runtime.connect(
'nckgahadagoaajjgafhacjanaoiihapd'
)
return {
connectResponse: {
exists: !!connectResponse,
toString1: connectResponse + '',
toString2: connectResponse.toString(),
nestedToString: connectResponse.onDisconnect.addListener + ''
},
disconnect: {
toString: connectResponse.disconnect + '',
noReturn: connectResponse.disconnect() === undefined
}
}
})
t.deepEqual(results, {
connectResponse: {
exists: true,
toString1: '[object Object]',
toString2: '[object Object]',
nestedToString: `function addListener() { [native code] }`
},
disconnect: {
toString: `function disconnect() { [native code] }`,
noReturn: true
}
})
})
// FIXME: This changed in more recent chrome versions
// test('stealth: error stack is fine', async t => {
// const puppeteer = addExtra(vanillaPuppeteer).use(
// Plugin({
// runOnInsecureOrigins: true // for testing
// })
// )
// const browser = await puppeteer.launch({ headless: true })
// const page = await browser.newPage()
// const result = await page.evaluate(() => {
// const catchErr = (fn, ...args) => {
// try {
// return fn.apply(this, args)
// } catch ({ name, message, stack }) {
// return {
// name,
// message,
// stack
// }
// }
// }
// return catchErr(chrome.runtime.connect, '').stack
// })
// /**
// * OK:
// TypeError: Error in invocation of runtime.connect(optional string extensionId, optional object connectInfo): chrome.runtime.connect() called from a webpage must specify an Extension ID (string) for its first argument.␊
// - at catchErr (__puppeteer_evaluation_script__:4:19)␊
// - at __puppeteer_evaluation_script__:18:12
// */
// t.is(result.split('\n').length, 3)
// })

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,33 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
- [sendMessageHandler()](#sendmessagehandler)
- [connectHandler()](#connecthandler)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/index.js#L13-L251)
- `opts` (optional, default `{}`)
**Extends: PuppeteerExtraPlugin**
Mock the `chrome.runtime` object if not available (e.g. when running headless) and on a secure site.
---
### [sendMessageHandler()](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/index.js#L80-L123)
Mock `chrome.runtime.sendMessage`
---
### [connectHandler()](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/index.js#L136-L210)
Mock `chrome.runtime.connect`
- **See: <https://developer.chrome.com/apps/runtime#method-connect>**
---

View File

@@ -0,0 +1,41 @@
{
"OnInstalledReason": {
"CHROME_UPDATE": "chrome_update",
"INSTALL": "install",
"SHARED_MODULE_UPDATE": "shared_module_update",
"UPDATE": "update"
},
"OnRestartRequiredReason": {
"APP_UPDATE": "app_update",
"OS_UPDATE": "os_update",
"PERIODIC": "periodic"
},
"PlatformArch": {
"ARM": "arm",
"ARM64": "arm64",
"MIPS": "mips",
"MIPS64": "mips64",
"X86_32": "x86-32",
"X86_64": "x86-64"
},
"PlatformNaclArch": {
"ARM": "arm",
"MIPS": "mips",
"MIPS64": "mips64",
"X86_32": "x86-32",
"X86_64": "x86-64"
},
"PlatformOs": {
"ANDROID": "android",
"CROS": "cros",
"LINUX": "linux",
"MAC": "mac",
"OPENBSD": "openbsd",
"WIN": "win"
},
"RequestUpdateCheckStatus": {
"NO_UPDATE": "no_update",
"THROTTLED": "throttled",
"UPDATE_AVAILABLE": "update_available"
}
}

View File

@@ -0,0 +1,47 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const argsToIgnore = [
'--disable-extensions',
'--disable-default-apps',
'--disable-component-extensions-with-background-pages'
]
/**
* A CDP driver like puppeteer can make use of various browser launch arguments that are
* adversarial to mimicking a regular browser and need to be stripped when launching the browser.
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/defaultArgs'
}
get requirements() {
return new Set(['runLast']) // So other plugins can modify launch options before
}
async beforeLaunch(options = {}) {
options.ignoreDefaultArgs = options.ignoreDefaultArgs || []
if (options.ignoreDefaultArgs === true) {
// that means the user explicitly wants to disable all default arguments
return
}
argsToIgnore.forEach(arg => {
if (options.ignoreDefaultArgs.includes(arg)) {
return
}
options.ignoreDefaultArgs.push(arg)
})
}
}
module.exports = function (pluginConfig) {
return new Plugin(pluginConfig)
}
module.exports.argsToIgnore = argsToIgnore

View File

@@ -0,0 +1,36 @@
const test = require('ava')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
const { argsToIgnore } = require('.')
test('vanilla: uses args to ignore', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const client =
typeof page._client === 'function' ? page._client() : page._client
const { arguments: launchArgs } = await client.send(
'Browser.getBrowserCommandLine'
)
const ok = argsToIgnore.every(arg => launchArgs.includes(arg))
if (!ok) {
console.log({ argsToIgnore, launchArgs })
}
t.is(ok, true)
})
test('stealth: does not use args to ignore', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const client =
typeof page._client === 'function' ? page._client() : page._client
const { arguments: launchArgs } = await client.send(
'Browser.getBrowserCommandLine'
)
const ok = argsToIgnore.every(arg => !launchArgs.includes(arg))
if (!ok) {
console.log({ argsToIgnore, launchArgs })
}
t.is(ok, true)
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,18 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/358246d5cc56bbb8800624128503482b8d7b426a/packages/puppeteer-extra-plugin-stealth/evasions/defaultArgs/index.js#L15-L41)
- `opts` (optional, default `{}`)
**Extends: PuppeteerExtraPlugin**
A CDP driver like puppeteer can make use of various browser launch arguments that are
adversarial to mimicking a regular browser and need to be stripped when launching the browser.
---

View File

@@ -0,0 +1,136 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const withUtils = require('../_utils/withUtils')
/**
* Fix for the HEADCHR_IFRAME detection (iframe.contentWindow.chrome), hopefully this time without breaking iframes.
* Note: Only `srcdoc` powered iframes cause issues due to a chromium bug:
*
* https://github.com/puppeteer/puppeteer/issues/1106
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/iframe.contentWindow'
}
get requirements() {
// Make sure `chrome.runtime` has ran, we use data defined by it (e.g. `window.chrome`)
return new Set(['runLast'])
}
async onPageCreated(page) {
await withUtils(page).evaluateOnNewDocument((utils, opts) => {
try {
// Adds a contentWindow proxy to the provided iframe element
const addContentWindowProxy = iframe => {
const contentWindowProxy = {
get(target, key) {
// Now to the interesting part:
// We actually make this thing behave like a regular iframe window,
// by intercepting calls to e.g. `.self` and redirect it to the correct thing. :)
// That makes it possible for these assertions to be correct:
// iframe.contentWindow.self === window.top // must be false
if (key === 'self') {
return this
}
// iframe.contentWindow.frameElement === iframe // must be true
if (key === 'frameElement') {
return iframe
}
// Intercept iframe.contentWindow[0] to hide the property 0 added by the proxy.
if (key === '0') {
return undefined
}
return Reflect.get(target, key)
}
}
if (!iframe.contentWindow) {
const proxy = new Proxy(window, contentWindowProxy)
Object.defineProperty(iframe, 'contentWindow', {
get() {
return proxy
},
set(newValue) {
return newValue // contentWindow is immutable
},
enumerable: true,
configurable: false
})
}
}
// Handles iframe element creation, augments `srcdoc` property so we can intercept further
const handleIframeCreation = (target, thisArg, args) => {
const iframe = target.apply(thisArg, args)
// We need to keep the originals around
const _iframe = iframe
const _srcdoc = _iframe.srcdoc
// Add hook for the srcdoc property
// We need to be very surgical here to not break other iframes by accident
Object.defineProperty(iframe, 'srcdoc', {
configurable: true, // Important, so we can reset this later
get: function() {
return _srcdoc
},
set: function(newValue) {
addContentWindowProxy(this)
// Reset property, the hook is only needed once
Object.defineProperty(iframe, 'srcdoc', {
configurable: false,
writable: false,
value: _srcdoc
})
_iframe.srcdoc = newValue
}
})
return iframe
}
// Adds a hook to intercept iframe creation events
const addIframeCreationSniffer = () => {
/* global document */
const createElementHandler = {
// Make toString() native
get(target, key) {
return Reflect.get(target, key)
},
apply: function(target, thisArg, args) {
const isIframe =
args && args.length && `${args[0]}`.toLowerCase() === 'iframe'
if (!isIframe) {
// Everything as usual
return target.apply(thisArg, args)
} else {
return handleIframeCreation(target, thisArg, args)
}
}
}
// All this just due to iframes with srcdoc bug
utils.replaceWithProxy(
document,
'createElement',
createElementHandler
)
}
// Let's go
addIframeCreationSniffer()
} catch (err) {
// console.warn(err)
}
})
}
}
module.exports = function(pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,448 @@
const test = require('ava')
const {
getVanillaFingerPrint,
getStealthFingerPrint,
dummyHTMLPath,
vanillaPuppeteer,
addExtra
} = require('../../test/util')
// const Plugin = require('.')
// NOTE: We're using the full plugin for testing here as `iframe.contentWindow` uses data set by `chrome.runtime`
const Plugin = require('puppeteer-extra-plugin-stealth')
// Fix CI issues with old versions
const isOldPuppeteerVersion = () => {
const version = process.env.PUPPETEER_VERSION
const isOld = version && (version === '1.9.0' || version === '1.6.2')
return isOld
}
test('vanilla: will be undefined', async t => {
const { iframeChrome } = await getVanillaFingerPrint()
t.is(iframeChrome, 'undefined')
})
test('stealth: will be object', async t => {
const { iframeChrome } = await getStealthFingerPrint(Plugin)
t.is(iframeChrome, 'object')
})
test('stealth: will not break iframes', async t => {
const browser = await addExtra(vanillaPuppeteer)
.use(Plugin())
.launch({ headless: true })
const page = await browser.newPage()
const testFuncReturnValue = 'TESTSTRING'
await page.evaluate(returnValue => {
const { document } = window // eslint-disable-line
const body = document.querySelector('body')
const iframe = document.createElement('iframe')
body.srcdoc = 'foobar'
body.appendChild(iframe)
iframe.contentWindow.mySuperFunction = () => returnValue
}, testFuncReturnValue)
const realReturn = await page.evaluate(
() => document.querySelector('iframe').contentWindow.mySuperFunction() // eslint-disable-line
)
await browser.close()
t.is(realReturn, 'TESTSTRING')
})
test('vanilla: will not have contentWindow[0]', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const zero = await page.evaluate(returnValue => {
const { document } = window // eslint-disable-line
const body = document.querySelector('body')
const iframe = document.createElement('iframe')
iframe.srcdoc = 'foobar'
body.appendChild(iframe)
return typeof iframe.contentWindow[0]
})
await browser.close()
t.is(zero, 'undefined')
})
test('stealth: will not have contentWindow[0]', async t => {
const browser = await addExtra(vanillaPuppeteer)
.use(Plugin())
.launch({ headless: true })
const page = await browser.newPage()
const zero = await page.evaluate(returnValue => {
const { document } = window // eslint-disable-line
const body = document.querySelector('body')
const iframe = document.createElement('iframe')
iframe.srcdoc = 'foobar'
body.appendChild(iframe)
return typeof iframe.contentWindow[0]
})
await browser.close()
t.is(zero, 'undefined')
})
test('vanilla: will not have chrome runtine in any frame', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
await page.goto('file://' + dummyHTMLPath)
const basiciframe = await page.evaluate(() => {
const el = document.createElement('iframe')
document.body.appendChild(el)
return el.contentWindow.chrome
})
const sandboxSOiframe = await page.evaluate(() => {
const el = document.createElement('iframe')
el.setAttribute('sandbox', 'allow-same-origin')
document.body.appendChild(el)
return el.contentWindow.chrome
})
const sandboxSOASiframe = await page.evaluate(() => {
const el = document.createElement('iframe')
el.setAttribute('sandbox', 'allow-same-origin allow-scripts')
document.body.appendChild(el)
return el.contentWindow.chrome
})
const srcdociframe = await page.evaluate(() => {
const el = document.createElement('iframe')
el.srcdoc = 'blank page, boys.'
document.body.appendChild(el)
return el.contentWindow.chrome
})
// console.log('basic iframe', basiciframe)
// console.log('sandbox same-origin iframe', sandboxSOiframe)
// console.log('sandbox same-origin&scripts iframe', sandboxSOASiframe)
// console.log('srcdoc iframe', srcdociframe)
await browser.close()
t.is(typeof basiciframe, 'undefined')
t.is(typeof sandboxSOiframe, 'undefined')
t.is(typeof sandboxSOASiframe, 'undefined')
t.is(typeof srcdociframe, 'undefined')
})
test('stealth: it will cover all frames including srcdoc', async t => {
// const browser = await vanillaPuppeteer.launch({ headless: false })
const browser = await addExtra(vanillaPuppeteer)
.use(Plugin())
.launch({ headless: true })
const page = await browser.newPage()
await page.goto('file://' + dummyHTMLPath)
const basiciframe = await page.evaluate(() => {
const el = document.createElement('iframe')
document.body.appendChild(el)
return el.contentWindow.chrome
})
const sandboxSOiframe = await page.evaluate(() => {
const el = document.createElement('iframe')
el.setAttribute('sandbox', 'allow-same-origin')
document.body.appendChild(el)
return el.contentWindow.chrome
})
const sandboxSOASiframe = await page.evaluate(() => {
const el = document.createElement('iframe')
el.setAttribute('sandbox', 'allow-same-origin allow-scripts')
document.body.appendChild(el)
return el.contentWindow.chrome
})
const srcdociframe = await page.evaluate(() => {
const el = document.createElement('iframe')
el.srcdoc = 'blank page, boys.'
document.body.appendChild(el)
return el.contentWindow.chrome
})
// console.log('basic iframe', basiciframe)
// console.log('sandbox same-origin iframe', sandboxSOiframe)
// console.log('sandbox same-origin&scripts iframe', sandboxSOASiframe)
// console.log('srcdoc iframe', srcdociframe)
await browser.close()
if (isOldPuppeteerVersion()) {
t.is(typeof basiciframe, 'object')
} else {
t.is(typeof basiciframe, 'object')
t.is(typeof sandboxSOiframe, 'object')
t.is(typeof sandboxSOASiframe, 'object')
t.is(typeof srcdociframe, 'object')
}
})
test('vanilla: will allow to define property contentWindow', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const iframe = await page.evaluate(() => {
const { document } = window // eslint-disable-line
const iframe = document.createElement('iframe')
iframe.srcdoc = 'foobar'
return Object.defineProperty(iframe, 'contentWindow', { value: 'baz' })
})
await browser.close()
t.is(typeof iframe, 'object')
})
// test('stealth: will allow to define property contentWindow', async t => {
// const browser = await addExtra(vanillaPuppeteer)
// .use(Plugin())
// .launch({ headless: true })
// const page = await browser.newPage()
// const iframe = await page.evaluate(() => {
// const { document } = window // eslint-disable-line
// const iframe = document.createElement('iframe')
// iframe.srcdoc = 'foobar'
// return Object.defineProperty(iframe, 'contentWindow', { value: 'baz' })
// })
// await browser.close()
// t.is(typeof iframe, 'object')
// })
test('vanilla: will return undefined for getOwnPropertyDescriptor of contentWindow', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const iframe = await page.evaluate(() => {
const { document } = window // eslint-disable-line
const iframe = document.createElement('iframe')
iframe.srcdoc = 'foobar'
return Object.getOwnPropertyDescriptor(iframe, 'contentWindow')
})
await browser.close()
t.is(iframe, undefined)
})
// test('stealth: will return undefined for getOwnPropertyDescriptor of contentWindow', async t => {
// const browser = await addExtra(vanillaPuppeteer)
// .use(Plugin())
// .launch({ headless: true })
// const page = await browser.newPage()
// const iframe = await page.evaluate(() => {
// const { document } = window // eslint-disable-line
// const iframe = document.createElement('iframe')
// iframe.srcdoc = 'foobar'
// return Object.getOwnPropertyDescriptor(iframe, 'contentWindow')
// })
// await browser.close()
// t.is(iframe, undefined)
// })
/* global HTMLIFrameElement */
test('stealth: it will emulate advanved contentWindow features correctly', async t => {
// const browser = await vanillaPuppeteer.launch({ headless: false })
const browser = await addExtra(vanillaPuppeteer)
.use(Plugin())
.launch({ headless: true })
const page = await browser.newPage()
await page.goto('file://' + dummyHTMLPath)
// page.on('console', msg => {
// console.log('Page console: ', msg.text())
// })
const results = await page.evaluate(() => {
const results = {}
const iframe = document.createElement('iframe')
iframe.srcdoc = 'page intentionally left blank' // Note: srcdoc
document.body.appendChild(iframe)
const basicIframe = document.createElement('iframe')
basicIframe.src = 'data:text/plain;charset=utf-8,foobar'
document.body.appendChild(iframe)
results.descriptors = (() => {
// Verify iframe prototype isn't touched
const descriptors = Object.getOwnPropertyDescriptors(
HTMLIFrameElement.prototype
)
return descriptors.contentWindow.get.toString()
})()
results.noProxySignature = (() => {
return iframe.srcdoc.toString.hasOwnProperty('[[IsRevoked]]') // eslint-disable-line
})()
results.doesExist = (() => {
// Verify iframe isn't remapped to main window
return !!iframe.contentWindow
})()
results.isNotAClone = (() => {
// Verify iframe isn't remapped to main window
return iframe.contentWindow !== window
})()
results.hasPlugins = (() => {
return iframe.contentWindow.navigator.plugins.length > 0
})()
results.hasSameNumberOfPlugins = (() => {
return (
window.navigator.plugins.length ===
iframe.contentWindow.navigator.plugins.length
)
})()
results.SelfIsNotWindow = (() => {
return iframe.contentWindow.self !== window
})()
results.SelfIsNotWindowTop = (() => {
return iframe.contentWindow.self !== window.top
})()
results.TopIsNotSame = (() => {
return iframe.contentWindow.top !== iframe.contentWindow
})()
results.FrameElementMatches = (() => {
return iframe.contentWindow.frameElement === iframe
})()
results.StackTraces = (() => {
try {
// eslint-disable-next-line
document['createElement'](0)
} catch (e) {
return e.stack
}
return false
})()
return results
})
await browser.close()
if (isOldPuppeteerVersion()) {
t.true(true)
return
}
t.is(results.descriptors, 'function get contentWindow() { [native code] }')
t.true(results.doesExist)
t.true(results.isNotAClone)
t.true(results.hasPlugins)
t.true(results.hasSameNumberOfPlugins)
t.true(results.SelfIsNotWindow)
t.true(results.SelfIsNotWindowTop)
t.true(results.TopIsNotSame)
t.false(results.StackTraces.includes(`at Object.apply`))
})
test('regression: new method will not break hcaptcha', async t => {
const browser = await addExtra(vanillaPuppeteer)
.use(Plugin())
.launch({ headless: true })
const page = await browser.newPage()
page.waitForTimeout = page.waitForTimeout || page.waitFor
await page.goto('https://democaptcha.com/demo-form-eng/hcaptcha.html', {
waitUntil: 'networkidle2'
})
await page.evaluate(() => {
window.hcaptcha.execute()
})
await page.waitForTimeout(2 * 1000)
const { hasChallengePopup } = await page.evaluate(() => {
const hasChallengePopup = !!document.querySelectorAll(
`div[style*='visible'] iframe[title*='hCaptcha challenge']`
).length
return { hasChallengePopup }
})
await browser.close()
t.true(hasChallengePopup)
})
test('regression: new method will not break recaptcha popup', async t => {
// const browser = await vanillaPuppeteer.launch({ headless: false })
const browser = await addExtra(vanillaPuppeteer)
.use(Plugin())
.launch({ headless: true })
const page = await browser.newPage()
page.waitForTimeout = page.waitForTimeout || page.waitFor
await page.goto('https://www.fbdemo.com/invisible-captcha/index.html', {
waitUntil: 'networkidle2'
})
await page.type('#tswname', 'foo')
await page.type('#tswemail', 'foo@foo.foo')
await page.type(
'#tswcomments',
'In the depth of winter, I finally learned that within me there lay an invincible summer.'
)
await page.click('#tswsubmit')
await page.waitForTimeout(1000)
const { hasRecaptchaPopup } = await page.evaluate(() => {
const hasRecaptchaPopup = !!document.querySelectorAll(
`iframe[title*="recaptcha challenge"]`
).length
return { hasRecaptchaPopup }
})
await browser.close()
t.true(hasRecaptchaPopup)
})
test('regression: old method indeed did break recaptcha popup', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
page.waitForTimeout = page.waitForTimeout || page.waitFor
// Old method
await page.evaluateOnNewDocument(() => {
// eslint-disable-next-line
Object.defineProperty(HTMLIFrameElement.prototype, 'contentWindow', {
get: function() {
return window
}
})
})
await page.goto('https://www.fbdemo.com/invisible-captcha/index.html', {
waitUntil: 'networkidle2'
})
await page.type('#tswname', 'foo')
await page.type('#tswemail', 'foo@foo.foo')
await page.type(
'#tswcomments',
'In the depth of winter, I finally learned that within me there lay an invincible summer.'
)
await page.click('#tswsubmit')
await page.waitForTimeout(1000)
const { hasRecaptchaPopup } = await page.evaluate(() => {
const hasRecaptchaPopup = !!document.querySelectorAll(
`iframe[title*="recaptcha challenge"]`
).length
return { hasRecaptchaPopup }
})
await browser.close()
t.false(hasRecaptchaPopup)
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,20 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow/index.js#L11-L125)
- `opts` (optional, default `{}`)
**Extends: PuppeteerExtraPlugin**
Fix for the HEADCHR_IFRAME detection (iframe.contentWindow.chrome), hopefully this time without breaking iframes.
Note: Only `srcdoc` powered iframes cause issues due to a chromium bug:
<https://github.com/puppeteer/puppeteer/issues/1106>
---

View File

@@ -0,0 +1,91 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const withUtils = require('../_utils/withUtils')
/**
* Fix Chromium not reporting "probably" to codecs like `videoEl.canPlayType('video/mp4; codecs="avc1.42E01E"')`.
* (Chromium doesn't support proprietary codecs, only Chrome does)
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/media.codecs'
}
async onPageCreated(page) {
await withUtils(page).evaluateOnNewDocument(utils => {
/**
* Input might look funky, we need to normalize it so e.g. whitespace isn't an issue for our spoofing.
*
* @example
* video/webm; codecs="vp8, vorbis"
* video/mp4; codecs="avc1.42E01E"
* audio/x-m4a;
* audio/ogg; codecs="vorbis"
* @param {String} arg
*/
const parseInput = arg => {
const [mime, codecStr] = arg.trim().split(';')
let codecs = []
if (codecStr && codecStr.includes('codecs="')) {
codecs = codecStr
.trim()
.replace(`codecs="`, '')
.replace(`"`, '')
.trim()
.split(',')
.filter(x => !!x)
.map(x => x.trim())
}
return {
mime,
codecStr,
codecs
}
}
const canPlayType = {
// Intercept certain requests
apply: function(target, ctx, args) {
if (!args || !args.length) {
return target.apply(ctx, args)
}
const { mime, codecs } = parseInput(args[0])
// This specific mp4 codec is missing in Chromium
if (mime === 'video/mp4') {
if (codecs.includes('avc1.42E01E')) {
return 'probably'
}
}
// This mimetype is only supported if no codecs are specified
if (mime === 'audio/x-m4a' && !codecs.length) {
return 'maybe'
}
// This mimetype is only supported if no codecs are specified
if (mime === 'audio/aac' && !codecs.length) {
return 'probably'
}
// Everything else as usual
return target.apply(ctx, args)
}
}
/* global HTMLMediaElement */
utils.replaceWithProxy(
HTMLMediaElement.prototype,
'canPlayType',
canPlayType
)
})
}
}
module.exports = function(pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,104 @@
const test = require('ava')
const {
getVanillaFingerPrint,
getStealthFingerPrint
} = require('../../test/util')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
test('vanilla: doesnt support proprietary codecs', async t => {
const { videoCodecs, audioCodecs } = await getVanillaFingerPrint()
t.deepEqual(videoCodecs, { ogg: 'probably', h264: '', webm: 'probably' })
t.deepEqual(audioCodecs, {
ogg: 'probably',
mp3: 'probably',
wav: 'probably',
m4a: '',
aac: ''
})
})
test('vanilla: will not have modifications', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
// https://datadome.co/bot-detection/client-side-detection-is-essential-for-bot-protection/
const test1 = await page.evaluate(() => {
const audioElt = document.createElement('audio')
return audioElt.canPlayType.toString()
})
t.is(test1, 'function canPlayType() { [native code] }')
const test2 = await page.evaluate(() => {
const audioElt = document.createElement('audio')
return audioElt.canPlayType.name
})
t.is(test2, 'canPlayType')
})
test('stealth: supports proprietary codecs', async t => {
const { videoCodecs, audioCodecs } = await getStealthFingerPrint(Plugin)
t.deepEqual(videoCodecs, {
ogg: 'probably',
h264: 'probably',
webm: 'probably'
})
t.deepEqual(audioCodecs, {
ogg: 'probably',
mp3: 'probably',
wav: 'probably',
m4a: 'maybe',
aac: 'probably'
})
})
test('stealth: will not leak modifications', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
// https://datadome.co/bot-detection/client-side-detection-is-essential-for-bot-protection/
const test1 = await page.evaluate(() => {
const audioElt = document.createElement('audio')
return audioElt.canPlayType.toString()
})
t.is(test1, 'function canPlayType() { [native code] }')
const test2 = await page.evaluate(() => {
const audioElt = document.createElement('audio')
return audioElt.canPlayType.name
})
t.is(test2, 'canPlayType')
// Double check the plugin is active and spoofing e.g. the aac codec results
const isWorkingTest = await page.evaluate(() => {
const audioElt = document.createElement('audio')
return audioElt.canPlayType('audio/aac') === 'probably' // empty in Chromium without stealth plugin
})
t.true(isWorkingTest)
})
test('vanilla: normal toString stuff', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const test1 = await page.evaluate(() => {
const audioElt = document.createElement('audio')
return audioElt.canPlayType.toString + ''
})
t.is(test1, 'function toString() { [native code] }')
})
test('stealth: will not leak toString stuff', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const test1 = await page.evaluate(() => {
const audioElt = document.createElement('audio')
return audioElt.canPlayType.toString + ''
})
t.is(test1, 'function toString() { [native code] }') // returns function () { [native code] }
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,39 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
- [parseInput(arg)](#parseinputarg)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/media.codecs/index.js#L12-L88)
- `opts` (optional, default `{}`)
**Extends: PuppeteerExtraPlugin**
Fix Chromium not reporting "probably" to codecs like `videoEl.canPlayType('video/mp4; codecs="avc1.42E01E"')`.
(Chromium doesn't support proprietary codecs, only Chrome does)
---
### [parseInput(arg)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/media.codecs/index.js#L33-L51)
- `arg` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**
Input might look funky, we need to normalize it so e.g. whitespace isn't an issue for our spoofing.
Example:
```javascript
video / webm
codecs = 'vp8, vorbis'
video / mp4
codecs = 'avc1.42E01E'
audio / x - m4a
audio / ogg
codecs = 'vorbis'
```
---

View File

@@ -0,0 +1,49 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const withUtils = require('../_utils/withUtils')
/**
* Set the hardwareConcurrency to 4 (optionally configurable with `hardwareConcurrency`)
*
* @see https://arh.antoinevastel.com/reports/stats/osName_hardwareConcurrency_report.html
*
* @param {Object} [opts] - Options
* @param {number} [opts.hardwareConcurrency] - The value to use in `navigator.hardwareConcurrency` (default: `4`)
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/navigator.hardwareConcurrency'
}
get defaults() {
return {
hardwareConcurrency: 4
}
}
async onPageCreated(page) {
await withUtils(page).evaluateOnNewDocument(
(utils, { opts }) => {
utils.replaceGetterWithProxy(
Object.getPrototypeOf(navigator),
'hardwareConcurrency',
utils.makeHandler().getterValue(opts.hardwareConcurrency)
)
},
{
opts: this.opts
}
)
}
}
module.exports = function (pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,59 @@
const test = require('ava')
const os = require('os')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const {
getVanillaFingerPrint,
getStealthFingerPrint
} = require('../../test/util')
const Plugin = require('.')
const fingerprintFn = page => page.evaluate('navigator.hardwareConcurrency')
test('vanilla: matches real core count', async t => {
const { pageFnResult } = await getVanillaFingerPrint(fingerprintFn)
t.is(pageFnResult, os.cpus().length)
})
test('stealth: default is set to 4', async t => {
const { pageFnResult } = await getStealthFingerPrint(Plugin, fingerprintFn)
t.is(pageFnResult, 4)
})
test('stealth: will override value correctly', async t => {
const { pageFnResult } = await getStealthFingerPrint(Plugin, fingerprintFn, {
hardwareConcurrency: 8
})
t.is(pageFnResult, 8)
})
test('stealth: does patch getters properly', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await page.evaluate(() => {
const hasInvocationError = (() => {
try {
// eslint-disable-next-line dot-notation
Object['seal'](Object.getPrototypeOf(navigator)['hardwareConcurrency'])
return false
} catch (err) {
return true
}
})()
return {
hasInvocationError,
toString: Object.getOwnPropertyDescriptor(
Object.getPrototypeOf(navigator),
'hardwareConcurrency'
).get.toString()
}
})
t.deepEqual(results, {
hasInvocationError: true,
toString: 'function get hardwareConcurrency() { [native code] }'
})
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,20 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/9534845cc95088e65c2d53bfb029263976fc9add/packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/index.js#L16-L37)
- `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`)
- `opts.hardwareConcurrency` **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)?** The value to use in `navigator.hardwareConcurrency` (default: `4`)
**Extends: PuppeteerExtraPlugin**
Set the hardwareConcurrency to 4 (optionally configurable with `hardwareConcurrency`)
- **See: <https://arh.antoinevastel.com/reports/stats/osName_hardwareConcurrency_report.html>**
---

View File

@@ -0,0 +1,48 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const withUtils = require('../_utils/withUtils')
/**
* Pass the Languages Test. Allows setting custom languages.
*
* @param {Object} [opts] - Options
* @param {Array<string>} [opts.languages] - The languages to use (default: `['en-US', 'en']`)
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/navigator.languages'
}
get defaults() {
return {
languages: [] // Empty default, otherwise this would be merged with user defined array override
}
}
async onPageCreated(page) {
await withUtils(page).evaluateOnNewDocument(
(utils, { opts }) => {
const languages = opts.languages.length
? opts.languages
: ['en-US', 'en']
utils.replaceGetterWithProxy(
Object.getPrototypeOf(navigator),
'languages',
utils.makeHandler().getterValue(Object.freeze([...languages]))
)
},
{
opts: this.opts
}
)
}
}
module.exports = function (pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,102 @@
const test = require('ava')
const {
getVanillaFingerPrint,
getStealthFingerPrint
} = require('../../test/util')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
// TODO: Vanilla seems fine, evasion obsolete?
// Note: We keep it around for now, as we will need this method in a fingerprinting plugin later anyway
test('vanilla: is array with en-US', async t => {
const { languages } = await getVanillaFingerPrint()
t.is(Array.isArray(languages), true)
t.is(languages[0], 'en-US')
})
test('vanilla: will not have modifications', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const test1 = await page.evaluate(
() => Object.getOwnPropertyDescriptor(navigator, 'languages') // Must be undefined if native
)
t.is(test1, undefined)
const test2 = await page.evaluate(
() => Object.getOwnPropertyNames(navigator) // Must be an empty array if native
)
t.false(test2.includes('languages'))
})
test('stealth: is array with en-US', async t => {
const { languages } = await getStealthFingerPrint(Plugin)
t.is(Array.isArray(languages), true)
t.is(languages[0], 'en-US')
})
test('stealth: customized value', async t => {
const { languages } = await getStealthFingerPrint(Plugin, null, {
languages: ['foo', 'bar']
})
t.deepEqual(languages, ['foo', 'bar'])
})
test('stealth: will not leak modifications', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const test1 = await page.evaluate(
() => Object.getOwnPropertyDescriptor(navigator, 'languages') // Must be undefined if native
)
t.is(test1, undefined)
const test2 = await page.evaluate(
() => Object.getOwnPropertyNames(navigator) // Must be an empty array if native
)
t.false(test2.includes('languages'))
})
test('stealth: does patch getters properly', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await page.evaluate(() => {
const hasInvocationError = (() => {
try {
// eslint-disable-next-line dot-notation
Object['seal'](Object.getPrototypeOf(navigator)['languages'])
return false
} catch (err) {
return true
}
})()
const hasPushError = (() => {
try {
// eslint-disable-next-line dot-notation
navigator.languages.push(null)
return false
} catch (err) {
return true
}
})()
return {
hasInvocationError,
hasPushError,
toString: Object.getOwnPropertyDescriptor(
Object.getPrototypeOf(navigator),
'languages'
).get.toString()
}
})
t.deepEqual(results, {
hasInvocationError: true,
hasPushError: true,
toString: 'function get languages() { [native code] }'
})
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,18 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.js#L11-L28)
- `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`)
- `opts.languages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)&lt;[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>?** The languages to use (default: `['en-US', 'en']`)
**Extends: PuppeteerExtraPlugin**
Pass the Languages Test. Allows setting custom languages.
---

View File

@@ -0,0 +1,70 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const withUtils = require('../_utils/withUtils')
/**
* Fix `Notification.permission` behaving weirdly in headless mode
*
* @see https://bugs.chromium.org/p/chromium/issues/detail?id=1052332
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/navigator.permissions'
}
/* global Notification Permissions PermissionStatus */
async onPageCreated(page) {
await withUtils(page).evaluateOnNewDocument((utils, opts) => {
const isSecure = document.location.protocol.startsWith('https')
// In headful on secure origins the permission should be "default", not "denied"
if (isSecure) {
utils.replaceGetterWithProxy(Notification, 'permission', {
apply() {
return 'default'
}
})
}
// Another weird behavior:
// On insecure origins in headful the state is "denied",
// whereas in headless it's "prompt"
if (!isSecure) {
const handler = {
apply(target, ctx, args) {
const param = (args || [])[0]
const isNotifications =
param && param.name && param.name === 'notifications'
if (!isNotifications) {
return utils.cache.Reflect.apply(...arguments)
}
return Promise.resolve(
Object.setPrototypeOf(
{
state: 'denied',
onchange: null
},
PermissionStatus.prototype
)
)
}
}
// Note: Don't use `Object.getPrototypeOf` here
utils.replaceWithProxy(Permissions.prototype, 'query', handler)
}
}, this.opts)
}
}
module.exports = function (pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,105 @@
/* global Notification */
const test = require('ava')
const {
getVanillaFingerPrint,
getStealthFingerPrint
} = require('../../test/util')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
test('vanilla: is prompt', async t => {
const { permissions } = await getVanillaFingerPrint()
t.deepEqual(permissions, {
permission: 'denied',
state: 'prompt' // this is WRONG behavior, it's "denied" in headful!
})
})
test('stealth: is denied', async t => {
const { permissions } = await getStealthFingerPrint(Plugin)
t.deepEqual(permissions, {
permission: 'denied',
state: 'denied' // this is FIXED behavior, it's "denied" in headful!
})
})
async function getNotificationPermission() {
const { state, onchange } = await navigator.permissions.query({
name: 'notifications'
})
return {
state,
onchange,
permission: Notification.permission
}
}
test('vanilla headful: as expected', async t => {
const puppeteer = addExtra(vanillaPuppeteer)
const browser = await puppeteer.launch({ headless: false })
const page = await browser.newPage()
const result = await page.evaluate(getNotificationPermission)
t.deepEqual(result, {
state: 'denied',
onchange: null,
permission: 'denied'
})
await page.goto('https://example.com', {
waitUntil: 'domcontentloaded'
})
const result2 = await page.evaluate(getNotificationPermission)
t.deepEqual(result2, {
state: 'prompt',
onchange: null,
permission: 'default'
})
})
test('vanilla headless: as expected', async t => {
const puppeteer = addExtra(vanillaPuppeteer)
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const result = await page.evaluate(getNotificationPermission)
t.deepEqual(result, {
state: 'prompt', // should be denied
onchange: null,
permission: 'denied'
})
await page.goto('https://example.com', {
waitUntil: 'domcontentloaded'
})
const result2 = await page.evaluate(getNotificationPermission)
t.deepEqual(result2, {
state: 'prompt',
onchange: null,
permission: 'denied' // should be default
})
})
test('stealth headless: as vanilla headful', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const result = await page.evaluate(getNotificationPermission)
t.deepEqual(result, {
state: 'denied',
onchange: null,
permission: 'denied'
})
await page.goto('https://example.com', {
waitUntil: 'domcontentloaded'
})
const result2 = await page.evaluate(getNotificationPermission)
t.deepEqual(result2, {
state: 'prompt',
onchange: null,
permission: 'default'
})
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,17 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/index.js#L12-L45)
- `opts` (optional, default `{}`)
**Extends: PuppeteerExtraPlugin**
Pass the Permissions Test.
---

View File

@@ -0,0 +1,48 @@
{
"mimeTypes": [
{
"type": "application/pdf",
"suffixes": "pdf",
"description": "",
"__pluginName": "Chrome PDF Viewer"
},
{
"type": "application/x-google-chrome-pdf",
"suffixes": "pdf",
"description": "Portable Document Format",
"__pluginName": "Chrome PDF Plugin"
},
{
"type": "application/x-nacl",
"suffixes": "",
"description": "Native Client Executable",
"__pluginName": "Native Client"
},
{
"type": "application/x-pnacl",
"suffixes": "",
"description": "Portable Native Client Executable",
"__pluginName": "Native Client"
}
],
"plugins": [
{
"name": "Chrome PDF Plugin",
"filename": "internal-pdf-viewer",
"description": "Portable Document Format",
"__mimeTypes": ["application/x-google-chrome-pdf"]
},
{
"name": "Chrome PDF Viewer",
"filename": "mhjfbmdgcfjbbpaeojofohoefgiehjai",
"description": "",
"__mimeTypes": ["application/pdf"]
},
{
"name": "Native Client",
"filename": "internal-nacl-plugin",
"description": "",
"__mimeTypes": ["application/x-nacl", "application/x-pnacl"]
}
]
}

View File

@@ -0,0 +1,50 @@
/**
* `navigator.{plugins,mimeTypes}` share similar custom functions to look up properties
*
* Note: This is meant to be run in the context of the page.
*/
module.exports.generateFunctionMocks = utils => (
proto,
itemMainProp,
dataArray
) => ({
/** Returns the MimeType object with the specified index. */
item: utils.createProxy(proto.item, {
apply(target, ctx, args) {
if (!args.length) {
throw new TypeError(
`Failed to execute 'item' on '${
proto[Symbol.toStringTag]
}': 1 argument required, but only 0 present.`
)
}
// Special behavior alert:
// - Vanilla tries to cast strings to Numbers (only integers!) and use them as property index lookup
// - If anything else than an integer (including as string) is provided it will return the first entry
const isInteger = args[0] && Number.isInteger(Number(args[0])) // Cast potential string to number first, then check for integer
// Note: Vanilla never returns `undefined`
return (isInteger ? dataArray[Number(args[0])] : dataArray[0]) || null
}
}),
/** Returns the MimeType object with the specified name. */
namedItem: utils.createProxy(proto.namedItem, {
apply(target, ctx, args) {
if (!args.length) {
throw new TypeError(
`Failed to execute 'namedItem' on '${
proto[Symbol.toStringTag]
}': 1 argument required, but only 0 present.`
)
}
return dataArray.find(mt => mt[itemMainProp] === args[0]) || null // Not `undefined`!
}
}),
/** Does nothing and shall return nothing */
refresh: proto.refresh
? utils.createProxy(proto.refresh, {
apply(target, ctx, args) {
return undefined
}
})
: undefined
})

View File

@@ -0,0 +1,101 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const utils = require('../_utils')
const withUtils = require('../_utils/withUtils')
const { generateMimeTypeArray } = require('./mimeTypes')
const { generatePluginArray } = require('./plugins')
const { generateMagicArray } = require('./magicArray')
const { generateFunctionMocks } = require('./functionMocks')
const data = require('./data.json')
/**
* In headless mode `navigator.mimeTypes` and `navigator.plugins` are empty.
* This plugin emulates both of these with functional mocks to match regular headful Chrome.
*
* Note: mimeTypes and plugins cross-reference each other, so it makes sense to do them at the same time.
*
* @see https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins/mimeTypes
* @see https://developer.mozilla.org/en-US/docs/Web/API/MimeTypeArray
* @see https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins/plugins
* @see https://developer.mozilla.org/en-US/docs/Web/API/PluginArray
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/navigator.plugins'
}
async onPageCreated(page) {
await withUtils(page).evaluateOnNewDocument(
(utils, { fns, data }) => {
fns = utils.materializeFns(fns)
// That means we're running headful
const hasPlugins = 'plugins' in navigator && navigator.plugins.length
if (hasPlugins) {
return // nothing to do here
}
const mimeTypes = fns.generateMimeTypeArray(utils, fns)(data.mimeTypes)
const plugins = fns.generatePluginArray(utils, fns)(data.plugins)
// Plugin and MimeType cross-reference each other, let's do that now
// Note: We're looping through `data.plugins` here, not the generated `plugins`
for (const pluginData of data.plugins) {
pluginData.__mimeTypes.forEach((type, index) => {
plugins[pluginData.name][index] = mimeTypes[type]
Object.defineProperty(plugins[pluginData.name], type, {
value: mimeTypes[type],
writable: false,
enumerable: false, // Not enumerable
configurable: true
})
Object.defineProperty(mimeTypes[type], 'enabledPlugin', {
value:
type === 'application/x-pnacl'
? mimeTypes['application/x-nacl'].enabledPlugin // these reference the same plugin, so we need to re-use the Proxy in order to avoid leaks
: new Proxy(plugins[pluginData.name], {}), // Prevent circular references
writable: false,
enumerable: false, // Important: `JSON.stringify(navigator.plugins)`
configurable: true
})
})
}
const patchNavigator = (name, value) =>
utils.replaceProperty(Object.getPrototypeOf(navigator), name, {
get() {
return value
}
})
patchNavigator('mimeTypes', mimeTypes)
patchNavigator('plugins', plugins)
// All done
},
{
// We pass some functions to evaluate to structure the code more nicely
fns: utils.stringifyFns({
generateMimeTypeArray,
generatePluginArray,
generateMagicArray,
generateFunctionMocks
}),
data
}
)
}
}
module.exports = function(pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,56 @@
const test = require('ava')
const {
getVanillaFingerPrint,
getStealthFingerPrint
} = require('../../test/util')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
test('vanilla: empty plugins, empty mimetypes', async t => {
const { plugins, mimeTypes } = await getVanillaFingerPrint()
t.is(plugins.length, 0)
t.is(mimeTypes.length, 0)
})
test('vanilla: will not have modifications', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const test1 = await page.evaluate(() => ({
mimeTypes: Object.getOwnPropertyDescriptor(navigator, 'mimeTypes'), // Must be undefined if native
plugins: Object.getOwnPropertyDescriptor(navigator, 'plugins') // Must be undefined if native
}))
t.is(test1.mimeTypes, undefined)
t.is(test1.plugins, undefined)
const test2 = await page.evaluate(
() => Object.getOwnPropertyNames(navigator) // Must be an empty array if native
)
t.false(test2.includes('plugins'))
})
test('stealth: has plugin, has mimetypes', async t => {
const { plugins, mimeTypes } = await getStealthFingerPrint(Plugin)
t.is(plugins.length, 3)
t.is(mimeTypes.length, 4)
})
test('stealth: will not leak modifications', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const test1 = await page.evaluate(() => ({
mimeTypes: Object.getOwnPropertyDescriptor(navigator, 'mimeTypes'), // Must be undefined if native
plugins: Object.getOwnPropertyDescriptor(navigator, 'plugins') // Must be undefined if native
}))
t.is(test1.mimeTypes, undefined)
t.is(test1.plugins, undefined)
const test2 = await page.evaluate(
() => Object.getOwnPropertyNames(navigator) // Must be an empty array if native
)
t.false(test2.includes('plugins'))
})

View File

@@ -0,0 +1,144 @@
/* global MimeType MimeTypeArray Plugin PluginArray */
/**
* Generate a convincing and functional MimeType or Plugin array from scratch.
* They're so similar that it makes sense to use a single generator here.
*
* Note: This is meant to be run in the context of the page.
*/
module.exports.generateMagicArray = (utils, fns) =>
function(
dataArray = [],
proto = MimeTypeArray.prototype,
itemProto = MimeType.prototype,
itemMainProp = 'type'
) {
// Quick helper to set props with the same descriptors vanilla is using
const defineProp = (obj, prop, value) =>
Object.defineProperty(obj, prop, {
value,
writable: false,
enumerable: false, // Important for mimeTypes & plugins: `JSON.stringify(navigator.mimeTypes)`
configurable: true
})
// Loop over our fake data and construct items
const makeItem = data => {
const item = {}
for (const prop of Object.keys(data)) {
if (prop.startsWith('__')) {
continue
}
defineProp(item, prop, data[prop])
}
return patchItem(item, data)
}
const patchItem = (item, data) => {
let descriptor = Object.getOwnPropertyDescriptors(item)
// Special case: Plugins have a magic length property which is not enumerable
// e.g. `navigator.plugins[i].length` should always be the length of the assigned mimeTypes
if (itemProto === Plugin.prototype) {
descriptor = {
...descriptor,
length: {
value: data.__mimeTypes.length,
writable: false,
enumerable: false,
configurable: true // Important to be able to use the ownKeys trap in a Proxy to strip `length`
}
}
}
// We need to spoof a specific `MimeType` or `Plugin` object
const obj = Object.create(itemProto, descriptor)
// Virtually all property keys are not enumerable in vanilla
const blacklist = [...Object.keys(data), 'length', 'enabledPlugin']
return new Proxy(obj, {
ownKeys(target) {
return Reflect.ownKeys(target).filter(k => !blacklist.includes(k))
},
getOwnPropertyDescriptor(target, prop) {
if (blacklist.includes(prop)) {
return undefined
}
return Reflect.getOwnPropertyDescriptor(target, prop)
}
})
}
const magicArray = []
// Loop through our fake data and use that to create convincing entities
dataArray.forEach(data => {
magicArray.push(makeItem(data))
})
// Add direct property access based on types (e.g. `obj['application/pdf']`) afterwards
magicArray.forEach(entry => {
defineProp(magicArray, entry[itemMainProp], entry)
})
// This is the best way to fake the type to make sure this is false: `Array.isArray(navigator.mimeTypes)`
const magicArrayObj = Object.create(proto, {
...Object.getOwnPropertyDescriptors(magicArray),
// There's one ugly quirk we unfortunately need to take care of:
// The `MimeTypeArray` prototype has an enumerable `length` property,
// but headful Chrome will still skip it when running `Object.getOwnPropertyNames(navigator.mimeTypes)`.
// To strip it we need to make it first `configurable` and can then overlay a Proxy with an `ownKeys` trap.
length: {
value: magicArray.length,
writable: false,
enumerable: false,
configurable: true // Important to be able to use the ownKeys trap in a Proxy to strip `length`
}
})
// Generate our functional function mocks :-)
const functionMocks = fns.generateFunctionMocks(utils)(
proto,
itemMainProp,
magicArray
)
// We need to overlay our custom object with a JS Proxy
const magicArrayObjProxy = new Proxy(magicArrayObj, {
get(target, key = '') {
// Redirect function calls to our custom proxied versions mocking the vanilla behavior
if (key === 'item') {
return functionMocks.item
}
if (key === 'namedItem') {
return functionMocks.namedItem
}
if (proto === PluginArray.prototype && key === 'refresh') {
return functionMocks.refresh
}
// Everything else can pass through as normal
return utils.cache.Reflect.get(...arguments)
},
ownKeys(target) {
// There are a couple of quirks where the original property demonstrates "magical" behavior that makes no sense
// This can be witnessed when calling `Object.getOwnPropertyNames(navigator.mimeTypes)` and the absense of `length`
// My guess is that it has to do with the recent change of not allowing data enumeration and this being implemented weirdly
// For that reason we just completely fake the available property names based on our data to match what regular Chrome is doing
// Specific issues when not patching this: `length` property is available, direct `types` props (e.g. `obj['application/pdf']`) are missing
const keys = []
const typeProps = magicArray.map(mt => mt[itemMainProp])
typeProps.forEach((_, i) => keys.push(`${i}`))
typeProps.forEach(propName => keys.push(propName))
return keys
},
getOwnPropertyDescriptor(target, prop) {
if (prop === 'length') {
return undefined
}
return Reflect.getOwnPropertyDescriptor(target, prop)
}
})
return magicArrayObjProxy
}

View File

@@ -0,0 +1,18 @@
/* global MimeType MimeTypeArray */
/**
* Generate a convincing and functional MimeTypeArray (with mime types) from scratch.
*
* Note: This is meant to be run in the context of the page.
*
* @see https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins/mimeTypes
* @see https://developer.mozilla.org/en-US/docs/Web/API/MimeTypeArray
*/
module.exports.generateMimeTypeArray = (utils, fns) => mimeTypesData => {
return fns.generateMagicArray(utils, fns)(
mimeTypesData,
MimeTypeArray.prototype,
MimeType.prototype,
'type'
)
}

View File

@@ -0,0 +1,208 @@
const test = require('ava')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
test('stealth: will have convincing mimeTypes', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await page.evaluate(() => {
// We need to help serializing the error or it won't survive being sent back from `page.evaluate`
const catchErr = function(fn, ...args) {
try {
return fn.apply(this, args)
} catch ({ name, message, stack }) {
return { name, message, stack, str: stack.split('\n')[0] }
}
}
return {
mimeTypes: {
exists: 'mimeTypes' in navigator,
isArray: Array.isArray(navigator.mimeTypes),
length: navigator.mimeTypes.length,
// value: navigator.mimeTypes,
toString: navigator.mimeTypes.toString(),
toStringProto: navigator.mimeTypes.__proto__.toString(), // eslint-disable-line no-proto
protoSymbol: navigator.mimeTypes.__proto__[Symbol.toStringTag], // eslint-disable-line no-proto
// valueOf: navigator.mimeTypes.valueOf(),
valueOfSame: navigator.mimeTypes.valueOf() === navigator.mimeTypes,
json: JSON.stringify(navigator.mimeTypes),
hasPropPush: 'push' in navigator.mimeTypes,
hasPropLength: 'length' in navigator.mimeTypes,
hasLengthDescriptor: !!Object.getOwnPropertyDescriptor(
navigator.mimeTypes,
'length'
),
propertyNames: JSON.stringify(
Object.getOwnPropertyNames(navigator.mimeTypes)
),
lengthInProps: Object.getOwnPropertyNames(navigator.mimeTypes).includes(
'length'
),
keys: JSON.stringify(Object.keys(navigator.mimeTypes)),
namedPropsAuthentic: (function() {
navigator.mimeTypes.alice = 'bob'
return navigator.mimeTypes.namedItem('alice') === null // true on chrome
})(),
loopResult: (function() {
let res = ''
for (var bK = 0; bK < window.navigator.mimeTypes.length; bK++)
bK === window.navigator.mimeTypes.length - 1
? (res += window.navigator.mimeTypes[bK].type)
: (res += window.navigator.mimeTypes[bK].type + ',')
return res
})()
},
namedItem: {
exists: 'namedItem' in navigator.mimeTypes,
toString: navigator.mimeTypes.namedItem.toString(),
resultNotFound: navigator.mimeTypes.namedItem('foo'),
resultFound: navigator.mimeTypes // eslint-disable-line no-proto
.namedItem('application/pdf')
.__proto__.toString(),
errors: {
// For whatever weird reason the normal context doesn't suffice, we need to bind this to `navigator.mimeTypes`
noArgs: catchErr.bind(navigator.mimeTypes)(
navigator.mimeTypes.namedItem
).str,
noStackLeaks: !catchErr
.bind(navigator.mimeTypes)(navigator.mimeTypes.namedItem)
.stack.includes(`.apply`),
protoCall: catchErr.bind(navigator.mimeTypes)(
navigator.mimeTypes.__proto__.namedItem // eslint-disable-line no-proto
).str
}
},
item: {
exists: 'item' in navigator.mimeTypes,
toString: navigator.mimeTypes.item.toString(),
resultNotFound: navigator.mimeTypes.item('madness').type,
resultNotFoundNumberString: navigator.mimeTypes.item('777'),
resultEmptyString: navigator.mimeTypes.item('').type,
resultByNumberString: navigator.mimeTypes.item('2').type,
resultByNumberStringZero: navigator.mimeTypes.item('0').type,
resultByNumber: navigator.mimeTypes.item(2).type,
resultNull: navigator.mimeTypes.item(null).type,
resultFound: navigator.mimeTypes.item('application/x-nacl').type,
resultBrackets: navigator.mimeTypes['application/x-pnacl'].type,
errors: {
// For whatever weird reason the normal context doesn't suffice, we need to bind this to `navigator.mimeTypes`
noArgs: catchErr.bind(navigator.mimeTypes)(navigator.mimeTypes.item)
.str,
noStackLeaks: !catchErr
.bind(navigator.mimeTypes)(navigator.mimeTypes.item)
.stack.includes(`.apply`),
protoCall: catchErr.bind(navigator.mimeTypes)(
navigator.mimeTypes.__proto__.item // eslint-disable-line no-proto
).str
}
}
}
})
t.deepEqual(results.mimeTypes, {
exists: true,
hasPropPush: false,
hasPropLength: true,
hasLengthDescriptor: false,
isArray: false,
json: `{"0":{},"1":{},"2":{},"3":{}}`,
keys: `["0","1","2","3"]`,
length: 4,
lengthInProps: false,
loopResult:
'application/pdf,application/x-google-chrome-pdf,application/x-nacl,application/x-pnacl',
namedPropsAuthentic: true,
propertyNames: `["0","1","2","3","application/pdf","application/x-google-chrome-pdf","application/x-nacl","application/x-pnacl"]`,
protoSymbol: 'MimeTypeArray',
toString: '[object MimeTypeArray]',
toStringProto: '[object MimeTypeArray]',
valueOfSame: true
})
t.deepEqual(results.namedItem, {
exists: true,
toString: 'function namedItem() { [native code] }',
resultFound: '[object MimeType]',
resultNotFound: null,
errors: {
noArgs:
"TypeError: Failed to execute 'namedItem' on 'MimeTypeArray': 1 argument required, but only 0 present.",
noStackLeaks: true,
protoCall: 'TypeError: Illegal invocation'
}
})
t.deepEqual(results.item, {
exists: true,
resultBrackets: 'application/x-pnacl',
resultByNumber: 'application/x-nacl',
resultByNumberString: 'application/x-nacl',
resultByNumberStringZero: 'application/pdf',
resultEmptyString: 'application/pdf',
resultFound: 'application/pdf',
resultNotFound: 'application/pdf',
resultNotFoundNumberString: null,
resultNull: 'application/pdf',
toString: 'function item() { [native code] }',
errors: {
noArgs:
"TypeError: Failed to execute 'item' on 'MimeTypeArray': 1 argument required, but only 0 present.",
noStackLeaks: true,
protoCall: 'TypeError: Illegal invocation'
}
})
})
test('stealth: will have convincing mimeType entry', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await page.evaluate(() => ({
mimeType: {
exists: !!navigator.mimeTypes[0],
toString: navigator.mimeTypes[0].toString(),
toStringProto: navigator.mimeTypes[0].__proto__.toString(), // eslint-disable-line no-proto
protoSymbol: navigator.mimeTypes[0].__proto__[Symbol.toStringTag], // eslint-disable-line no-proto
enabledPlugin: !!navigator.mimeTypes[0].enabledPlugin, // should not throw
enabledPlugin2: !!navigator.mimeTypes['application/pdf'].enabledPlugin, // should not throw
enabledPlugins: !!navigator.mimeTypes[0].enabledPlugins, // regression: should not exist (anymore)
pdfPlugin: JSON.stringify(
navigator.mimeTypes['application/pdf'].enabledPlugin
),
length: !!navigator.mimeTypes[0].length, // should not throw and return mimeTypes length
lengthDescriptor: !!Object.getOwnPropertyDescriptor(
navigator.mimeTypes[0],
'length'
),
json: JSON.stringify(navigator.mimeTypes[0]),
propertyNames: JSON.stringify(
Object.getOwnPropertyNames(navigator.mimeTypes[0])
),
nested:
navigator.mimeTypes['application/pdf'].enabledPlugin[0].enabledPlugin[0]
.enabledPlugin[0].enabledPlugin[0].enabledPlugin[0].suffixes
}
}))
t.deepEqual(results.mimeType, {
exists: true,
protoSymbol: 'MimeType',
toString: '[object MimeType]',
toStringProto: '[object MimeType]',
enabledPlugin: true,
enabledPlugin2: true,
enabledPlugins: false,
pdfPlugin: '{"0":{}}',
length: false,
lengthDescriptor: false,
json: '{}',
propertyNames: '[]',
nested: 'pdf'
})
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,18 @@
/* global Plugin PluginArray */
/**
* Generate a convincing and functional PluginArray (with plugins) from scratch.
*
* Note: This is meant to be run in the context of the page.
*
* @see https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins/plugins
* @see https://developer.mozilla.org/en-US/docs/Web/API/PluginArray
*/
module.exports.generatePluginArray = (utils, fns) => pluginsData => {
return fns.generateMagicArray(utils, fns)(
pluginsData,
PluginArray.prototype,
Plugin.prototype,
'name'
)
}

View File

@@ -0,0 +1,184 @@
const test = require('ava')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
test('stealth: will have convincing plugins', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await page.evaluate(() => {
// We need to help serializing the error or it won't survive being sent back from `page.evaluate`
const catchErr = function(fn, ...args) {
try {
return fn.apply(this, args)
} catch ({ name, message, stack }) {
return { name, message, stack, str: stack.split('\n')[0] }
}
}
return {
plugins: {
exists: 'plugins' in navigator,
isArray: Array.isArray(navigator.plugins),
length: navigator.plugins.length,
// value: navigator.plugins,
toString: navigator.plugins.toString(),
toStringProto: navigator.plugins.__proto__.toString(), // eslint-disable-line no-proto
protoSymbol: navigator.plugins.__proto__[Symbol.toStringTag], // eslint-disable-line no-proto
// valueOf: navigator.plugins.valueOf(),
valueOfSame: navigator.plugins.valueOf() === navigator.plugins,
json: JSON.stringify(navigator.plugins),
hasPropPush: 'push' in navigator.plugins,
hasPropLength: 'length' in navigator.plugins,
hasLengthDescriptor: !!Object.getOwnPropertyDescriptor(
navigator.plugins,
'length'
),
propertyNames: JSON.stringify(
Object.getOwnPropertyNames(navigator.plugins)
),
lengthInProps: Object.getOwnPropertyNames(navigator.plugins).includes(
'length'
),
keys: JSON.stringify(Object.keys(navigator.plugins)),
loopResult: [...navigator.plugins].map(p => p.name).join(',')
},
namedItem: {
exists: 'namedItem' in navigator.plugins,
toString: navigator.plugins.namedItem.toString(),
resultNotFound: navigator.plugins.namedItem('foo'),
resultFound: navigator.plugins // eslint-disable-line no-proto
.namedItem('Chrome PDF Viewer')
.__proto__.toString(),
errors: {
// For whatever weird reason the normal context doesn't suffice, we need to bind this to `navigator.plugins`
noArgs: catchErr.bind(navigator.plugins)(navigator.plugins.namedItem)
.str,
noStackLeaks: !catchErr
.bind(navigator.plugins)(navigator.plugins.namedItem)
.stack.includes(`.apply`),
protoCall: catchErr.bind(navigator.plugins)(
navigator.plugins.__proto__.namedItem // eslint-disable-line no-proto
).str
}
},
item: {
exists: 'item' in navigator.plugins,
toString: navigator.plugins.item.toString(),
resultNotFound: navigator.plugins.item('madness').name,
resultNotFoundNumberString: navigator.plugins.item('777'),
resultEmptyString: navigator.plugins.item('').name,
resultByNumberString: navigator.plugins.item('2').name,
resultByNumberStringZero: navigator.plugins.item('0').name,
resultByNumber: navigator.plugins.item(2).name,
resultNull: navigator.plugins.item(null).name,
resultFound: navigator.plugins.item('application/x-nacl').name,
errors: {
// For whatever weird reason the normal context doesn't suffice, we need to bind this to `navigator.plugins`
noArgs: catchErr.bind(navigator.plugins)(navigator.plugins.item).str,
noStackLeaks: !catchErr
.bind(navigator.plugins)(navigator.plugins.item)
.stack.includes(`.apply`),
protoCall: catchErr.bind(navigator.plugins)(
navigator.plugins.__proto__.item // eslint-disable-line no-proto
).str
}
}
}
})
t.deepEqual(results.plugins, {
exists: true,
hasPropLength: true,
hasLengthDescriptor: false,
hasPropPush: false,
isArray: false,
json: `{"0":{"0":{}},"1":{"0":{}},"2":{"0":{},"1":{}}}`,
keys: `["0","1","2"]`,
length: 3,
lengthInProps: false,
loopResult: 'Chrome PDF Plugin,Chrome PDF Viewer,Native Client',
propertyNames: `["0","1","2","Chrome PDF Plugin","Chrome PDF Viewer","Native Client"]`,
protoSymbol: 'PluginArray',
toString: '[object PluginArray]',
toStringProto: '[object PluginArray]',
valueOfSame: true
})
t.deepEqual(results.namedItem, {
exists: true,
toString: 'function namedItem() { [native code] }',
resultFound: '[object Plugin]',
resultNotFound: null,
errors: {
noArgs:
"TypeError: Failed to execute 'namedItem' on 'PluginArray': 1 argument required, but only 0 present.",
noStackLeaks: true,
protoCall: 'TypeError: Illegal invocation'
}
})
t.deepEqual(results.item, {
exists: true,
resultByNumber: 'Native Client',
resultByNumberString: 'Native Client',
resultByNumberStringZero: 'Chrome PDF Plugin',
resultEmptyString: 'Chrome PDF Plugin',
resultFound: 'Chrome PDF Plugin',
resultNotFound: 'Chrome PDF Plugin',
resultNotFoundNumberString: null,
resultNull: 'Chrome PDF Plugin',
toString: 'function item() { [native code] }',
errors: {
noArgs:
"TypeError: Failed to execute 'item' on 'PluginArray': 1 argument required, but only 0 present.",
noStackLeaks: true,
protoCall: 'TypeError: Illegal invocation'
}
})
})
test('stealth: will have convincing plugin entry', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await page.evaluate(() => ({
plugins: {
exists: !!navigator.plugins[0],
toString: navigator.plugins[0].toString(),
toStringProto: navigator.plugins[0].__proto__.toString(), // eslint-disable-line no-proto
protoSymbol: navigator.plugins[0].__proto__[Symbol.toStringTag], // eslint-disable-line no-proto
length: navigator.plugins[0].length, // should not throw and return mimeTypes length
lengthDescriptor: Object.getOwnPropertyDescriptor(
navigator.plugins[0],
'length'
)
},
plugin: {
mtIndex: !!navigator.plugins[0][0], // mimeType should be accessible through index
mtNamed: !!navigator.plugins[0]['application/x-google-chrome-pdf'], // mimeType should be accessible through name
json: JSON.stringify(navigator.plugins[0]),
propertyNames: JSON.stringify(
Object.getOwnPropertyNames(navigator.plugins[0])
)
}
}))
t.deepEqual(results.plugins, {
exists: true,
protoSymbol: 'Plugin',
toString: '[object Plugin]',
toStringProto: '[object Plugin]',
length: 1
})
t.deepEqual(results.plugin, {
mtIndex: true,
mtNamed: true,
json: '{"0":{}}',
propertyNames: '["0","application/x-google-chrome-pdf"]'
})
})

View File

@@ -0,0 +1,25 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/index.js#L26-L88)
- `opts` (optional, default `{}`)
**Extends: PuppeteerExtraPlugin**
In headless mode `navigator.mimeTypes` and `navigator.plugins` are empty.
This plugin emulates both of these with functional mocks to match regular headful Chrome.
Note: mimeTypes and plugins cross-reference each other, so it makes sense to do them at the same time.
- **See: <https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins/mimeTypes>**
- **See: <https://developer.mozilla.org/en-US/docs/Web/API/MimeTypeArray>**
- **See: <https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins/plugins>**
- **See: <https://developer.mozilla.org/en-US/docs/Web/API/PluginArray>**
---

View File

@@ -0,0 +1,66 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const withUtils = require('../_utils/withUtils')
/**
* By default puppeteer will have a fixed `navigator.vendor` property.
*
* This plugin makes it possible to change this property.
*
* @example
* const puppeteer = require("puppeteer-extra")
*
* const StealthPlugin = require("puppeteer-extra-plugin-stealth")
* const stealth = StealthPlugin()
* // Remove this specific stealth plugin from the default set
* stealth.enabledEvasions.delete("navigator.vendor")
* puppeteer.use(stealth)
*
* // Stealth plugins are just regular `puppeteer-extra` plugins and can be added as such
* const NavigatorVendorPlugin = require("puppeteer-extra-plugin-stealth/evasions/navigator.vendor")
* const nvp = NavigatorVendorPlugin({ vendor: 'Apple Computer, Inc.' }) // Custom vendor
* puppeteer.use(nvp)
*
* @param {Object} [opts] - Options
* @param {string} [opts.vendor] - The vendor to use in `navigator.vendor` (default: `Google Inc.`)
*
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/navigator.vendor'
}
get defaults() {
return {
vendor: 'Google Inc.'
}
}
async onPageCreated(page) {
this.debug('onPageCreated', {
opts: this.opts
})
await withUtils(page).evaluateOnNewDocument(
(utils, { opts }) => {
utils.replaceGetterWithProxy(
Object.getPrototypeOf(navigator),
'vendor',
utils.makeHandler().getterValue(opts.vendor)
)
},
{
opts: this.opts
}
)
} // onPageCreated
}
const defaultExport = opts => new Plugin(opts)
module.exports = defaultExport

View File

@@ -0,0 +1,69 @@
const test = require('ava')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
test('vanilla: navigator.vendor is always Google Inc.', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const vendor = await page.evaluate(() => navigator.vendor)
t.is(vendor, 'Google Inc.')
})
test('stealth: navigator.vendor set to custom value', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(
Plugin({ vendor: 'Apple Computer, Inc.' })
)
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const vendor = await page.evaluate(() => navigator.vendor)
t.is(vendor, 'Apple Computer, Inc.')
})
test('stealth: will not leak modifications', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const test1 = await page.evaluate(
() => Object.getOwnPropertyDescriptor(navigator, 'vendor') // Must be undefined if native
)
t.is(test1, undefined)
const test2 = await page.evaluate(
() => Object.getOwnPropertyNames(navigator) // Must be an empty array if native
)
t.false(test2.includes('vendor'))
})
test('stealth: does patch getters properly', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const results = await page.evaluate(() => {
const hasInvocationError = (() => {
try {
// eslint-disable-next-line dot-notation
Object['seal'](Object.getPrototypeOf(navigator)['vendor'])
return false
} catch (err) {
return true
}
})()
return {
hasInvocationError,
toString: Object.getOwnPropertyDescriptor(
Object.getPrototypeOf(navigator),
'vendor'
).get.toString()
}
})
t.deepEqual(results, {
hasInvocationError: true,
toString: 'function get vendor() { [native code] }'
})
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,37 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.vendor/index.js#L28-L55)
- `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`)
- `opts.vendor` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The vendor to use in `navigator.vendor` (default: `Google Inc.`)
**Extends: PuppeteerExtraPlugin**
By default puppeteer will have a fixed `navigator.vendor` property.
This plugin makes it possible to change this property.
Example:
```javascript
const puppeteer = require('puppeteer-extra')
const StealthPlugin = require('puppeteer-extra-plugin-stealth')
const stealth = StealthPlugin()
// Remove this specific stealth plugin from the default set
stealth.enabledEvasions.delete('navigator.vendor')
puppeteer.use(stealth)
// Stealth plugins are just regular `puppeteer-extra` plugins and can be added as such
const NavigatorVendorPlugin = require('puppeteer-extra-plugin-stealth/evasions/navigator.vendor')
const nvp = NavigatorVendorPlugin({ vendor: 'Apple Computer, Inc.' }) // Custom vendor
puppeteer.use(nvp)
```
---

View File

@@ -0,0 +1,48 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
/**
* Pass the Webdriver Test.
* Will delete `navigator.webdriver` property.
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/navigator.webdriver'
}
async onPageCreated(page) {
await page.evaluateOnNewDocument(() => {
if (navigator.webdriver === false) {
// Post Chrome 89.0.4339.0 and already good
} else if (navigator.webdriver === undefined) {
// Pre Chrome 89.0.4339.0 and already good
} else {
// Pre Chrome 88.0.4291.0 and needs patching
delete Object.getPrototypeOf(navigator).webdriver
}
})
}
// Post Chrome 88.0.4291.0
// Note: this will add an infobar to Chrome with a warning that an unsupported flag is set
// To remove this bar on Linux, run: mkdir -p /etc/opt/chrome/policies/managed && echo '{ "CommandLineFlagSecurityWarningsEnabled": false }' > /etc/opt/chrome/policies/managed/managed_policies.json
async beforeLaunch(options) {
// If disable-blink-features is already passed, append the AutomationControlled switch
const idx = options.args.findIndex((arg) => arg.startsWith('--disable-blink-features='));
if (idx !== -1) {
const arg = options.args[idx];
options.args[idx] = `${arg},AutomationControlled`;
} else {
options.args.push('--disable-blink-features=AutomationControlled');
}
}
}
module.exports = function(pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,44 @@
const test = require('ava')
const { vanillaPuppeteer, addExtra, compareLooseVersionStrings } = require('../../test/util')
const Plugin = require('.')
function getExpectedValue(looseVersionString) {
if (compareLooseVersionStrings(looseVersionString, '89.0.4339.0') >= 0) {
return false
} else {
return undefined
}
}
test('vanilla: navigator.webdriver is defined', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const data = await page.evaluate(() => navigator.webdriver)
t.is(data, true)
})
test('stealth: navigator.webdriver is undefined', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const data = await page.evaluate(() => navigator.webdriver)
// XXX: launch this test multiple times with browsers of different versions?
t.is(data, getExpectedValue(await browser.version()))
})
// https://github.com/berstend/puppeteer-extra/pull/130
test('stealth: regression: wont kill other navigator methods', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
try {
const data = await page.evaluate(() => navigator.javaEnabled())
t.is(data, false)
} catch (err) {
t.is(err, undefined)
}
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,18 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.webdriver/index.js#L9-L23)
- `opts` (optional, default `{}`)
**Extends: PuppeteerExtraPlugin**
Pass the Webdriver Test.
Will delete `navigator.webdriver` property.
---

View File

@@ -0,0 +1,13 @@
# puppeteer-extra-plugin-stealth/evasions
Various detection evasion plugins for `puppeteer-extra-plugin-stealth`.
You can bypass the main module and require specific evasion plugins yourself, if you wish to do so:
```es6
puppeteer.use(
require('puppeteer-extra-plugin-stealth/evasions/console.debug')()
)
```
If you want to add a new evasion technique I suggest you look at the [template](./_template/) to kickstart things.

View File

@@ -0,0 +1,37 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title>Page Title</title>
<meta name="viewport" content="width=device-width, initial-scale=1" />
</head>
<body>
<h1 id="result">Please use `document.querySelector`..</h1>
<script>
function test() {
const err = new Error('Test Error')
const isPptr = err.stack
.toString()
.includes('puppeteer_evaluation_script')
document.getElementById('result').innerHTML = isPptr ? 'FAIL' : 'PASS'
console.log({ err, isPptr })
}
function overrideFunction(item) {
item.obj[item.propName] = (function(orig) {
return function() {
test()
return orig.apply(this, arguments)
}
})(item.obj[item.propName])
}
overrideFunction({
propName: 'querySelector',
obj: document
})
</script>
</body>
</html>

View File

@@ -0,0 +1,83 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
/**
* Strip sourceURL from scripts injected by puppeteer.
* It can be used to identify the presence of pptr via stacktraces.
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/sourceurl'
}
async onPageCreated(page) {
const client =
page && typeof page._client === 'function' ? page._client() : page._client
if (!client) {
this.debug('Warning, missing properties to intercept CDP.', { page })
return
}
// Intercept CDP commands and strip identifying and unnecessary sourceURL
// https://github.com/puppeteer/puppeteer/blob/9b3005c105995cd267fdc7fb95b78aceab82cf0e/new-docs/puppeteer.cdpsession.md
const debug = this.debug
client.send = (function(originalMethod, context) {
return async function() {
const [method, paramArgs] = arguments || []
const next = async () => {
try {
return await originalMethod.apply(context, [method, paramArgs])
} catch (error) {
// This seems to happen sometimes when redirects cause other outstanding requests to be cut short
if (
error instanceof Error &&
error.message.includes(
`Protocol error (Network.getResponseBody): No resource with given identifier found`
)
) {
debug(
`Caught and ignored an error about a missing network resource.`,
{ error }
)
} else {
throw error
}
}
}
if (!method || !paramArgs) {
return next()
}
// To find the methods/props in question check `_evaluateInternal` at:
// https://github.com/puppeteer/puppeteer/blob/main/src/common/ExecutionContext.ts#L186
const methodsToPatch = {
'Runtime.evaluate': 'expression',
'Runtime.callFunctionOn': 'functionDeclaration'
}
const SOURCE_URL_SUFFIX =
'//# sourceURL=__puppeteer_evaluation_script__'
if (!methodsToPatch[method] || !paramArgs[methodsToPatch[method]]) {
return next()
}
debug('Stripping sourceURL', { method })
paramArgs[methodsToPatch[method]] = paramArgs[
methodsToPatch[method]
].replace(SOURCE_URL_SUFFIX, '')
return next()
}
})(client.send, client)
}
}
module.exports = function(pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,54 @@
const test = require('ava')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
const TEST_HTML_FILE = require('path').join(__dirname, './_fixtures/test.html')
test('vanilla: sourceurl is leaking', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
await page.goto('file://' + TEST_HTML_FILE, { waitUntil: 'load' })
// Trigger test
await page.$('title')
const result = await page.evaluate(
() => document.querySelector('#result').innerText
)
t.is(result, 'FAIL')
const result2 = await page.evaluate(() => {
try {
Function.prototype.toString.apply({})
} catch (err) {
return err.stack
}
})
t.true(result2.includes('__puppeteer_evaluation_script'))
})
test('stealth: sourceurl is not leaking', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
await page.goto('file://' + TEST_HTML_FILE, { waitUntil: 'load' })
// Trigger test
await page.$('title')
const result = await page.evaluate(
() => document.querySelector('#result').innerText
)
t.is(result, 'PASS')
const result2 = await page.evaluate(() => {
try {
Function.prototype.toString.apply({})
} catch (err) {
return err.stack
}
})
t.false(result2.includes('__puppeteer_evaluation_script'))
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,18 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/sourceurl/index.js#L9-L58)
- `opts` (optional, default `{}`)
**Extends: PuppeteerExtraPlugin**
Strip sourceURL from scripts injected by puppeteer.
It can be used to identify the presence of pptr via stacktraces.
---

View File

@@ -0,0 +1,208 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
/**
* Fixes the UserAgent info (composed of UA string, Accept-Language, Platform, and UA hints).
*
* If you don't provide any values this plugin will default to using the regular UserAgent string (while stripping the headless part).
* Default language is set to "en-US,en", the other settings match the UserAgent string.
* If you are running on Linux, it will mask the settins to look like Windows. This behavior can be disabled with the `maskLinux` option.
*
* By default puppeteer will not set a `Accept-Language` header in headless:
* It's (theoretically) possible to fix that using either `page.setExtraHTTPHeaders` or a `--lang` launch arg.
* Unfortunately `page.setExtraHTTPHeaders` will lowercase everything and launch args are not always available. :)
*
* In addition, the `navigator.platform` property is always set to the host value, e.g. `Linux` which makes detection very easy.
*
* Note: You cannot use the regular `page.setUserAgent()` puppeteer call in your code,
* as it will reset the language and platform values you set with this plugin.
*
* @example
* const puppeteer = require("puppeteer-extra")
*
* const StealthPlugin = require("puppeteer-extra-plugin-stealth")
* const stealth = StealthPlugin()
* // Remove this specific stealth plugin from the default set
* stealth.enabledEvasions.delete("user-agent-override")
* puppeteer.use(stealth)
*
* // Stealth plugins are just regular `puppeteer-extra` plugins and can be added as such
* const UserAgentOverride = require("puppeteer-extra-plugin-stealth/evasions/user-agent-override")
* // Define custom UA and locale
* const ua = UserAgentOverride({ userAgent: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)", locale: "de-DE,de" })
* puppeteer.use(ua)
*
* @param {Object} [opts] - Options
* @param {string} [opts.userAgent] - The user agent to use (default: browser.userAgent())
* @param {string} [opts.locale] - The locale to use in `Accept-Language` header and in `navigator.languages` (default: `en-US,en`)
* @param {boolean} [opts.maskLinux] - Wether to hide Linux as platform in the user agent or not - true by default
*
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
this._headless = false
}
get name() {
return 'stealth/evasions/user-agent-override'
}
get dependencies() {
return new Set(['user-preferences'])
}
get defaults() {
return {
userAgent: null,
locale: 'en-US,en',
maskLinux: true
}
}
async onPageCreated(page) {
// Determine the full user agent string, strip the "Headless" part
let ua =
this.opts.userAgent ||
(await page.browser().userAgent()).replace('HeadlessChrome/', 'Chrome/')
if (
this.opts.maskLinux &&
ua.includes('Linux') &&
!ua.includes('Android') // Skip Android user agents since they also contain Linux
) {
ua = ua.replace(/\(([^)]+)\)/, '(Windows NT 10.0; Win64; x64)') // Replace the first part in parentheses with Windows data
}
// Full version number from Chrome
const uaVersion = ua.includes('Chrome/')
? ua.match(/Chrome\/([\d|.]+)/)[1]
: (await page.browser().version()).match(/\/([\d|.]+)/)[1]
// Get platform identifier (short or long version)
const _getPlatform = (extended = false) => {
if (ua.includes('Mac OS X')) {
return extended ? 'Mac OS X' : 'MacIntel'
} else if (ua.includes('Android')) {
return 'Android'
} else if (ua.includes('Linux')) {
return 'Linux'
} else {
return extended ? 'Windows' : 'Win32'
}
}
// Source in C++: https://source.chromium.org/chromium/chromium/src/+/master:components/embedder_support/user_agent_utils.cc;l=55-100
const _getBrands = () => {
const seed = uaVersion.split('.')[0] // the major version number of Chrome
const order = [
[0, 1, 2],
[0, 2, 1],
[1, 0, 2],
[1, 2, 0],
[2, 0, 1],
[2, 1, 0]
][seed % 6]
const escapedChars = [' ', ' ', ';']
const greaseyBrand = `${escapedChars[order[0]]}Not${
escapedChars[order[1]]
}A${escapedChars[order[2]]}Brand`
const greasedBrandVersionList = []
greasedBrandVersionList[order[0]] = {
brand: greaseyBrand,
version: '99'
}
greasedBrandVersionList[order[1]] = {
brand: 'Chromium',
version: seed
}
greasedBrandVersionList[order[2]] = {
brand: 'Google Chrome',
version: seed
}
return greasedBrandVersionList
}
// Return OS version
const _getPlatformVersion = () => {
if (ua.includes('Mac OS X ')) {
return ua.match(/Mac OS X ([^)]+)/)[1]
} else if (ua.includes('Android ')) {
return ua.match(/Android ([^;]+)/)[1]
} else if (ua.includes('Windows ')) {
return ua.match(/Windows .*?([\d|.]+);?/)[1]
} else {
return ''
}
}
// Get architecture, this seems to be empty on mobile and x86 on desktop
const _getPlatformArch = () => (_getMobile() ? '' : 'x86')
// Return the Android model, empty on desktop
const _getPlatformModel = () =>
_getMobile() ? ua.match(/Android.*?;\s([^)]+)/)[1] : ''
const _getMobile = () => ua.includes('Android')
const override = {
userAgent: ua,
platform: _getPlatform(),
userAgentMetadata: {
brands: _getBrands(),
fullVersion: uaVersion,
platform: _getPlatform(true),
platformVersion: _getPlatformVersion(),
architecture: _getPlatformArch(),
model: _getPlatformModel(),
mobile: _getMobile()
}
}
// In case of headless, override the acceptLanguage in CDP.
// This is not preferred, as it messed up the header order.
// On headful, we set the user preference language setting instead.
if (this._headless) {
override.acceptLanguage = this.opts.locale || 'en-US,en'
}
this.debug('onPageCreated - Will set these user agent options', {
override,
opts: this.opts
})
const client =
typeof page._client === 'function' ? page._client() : page._client
client.send('Network.setUserAgentOverride', override)
}
async beforeLaunch(options) {
// Check if launched headless
this._headless = options.headless
}
async beforeConnect() {
// Treat browsers using connect() as headless browsers
this._headless = true
}
get data() {
return [
{
name: 'userPreferences',
value: {
intl: { accept_languages: this.opts.locale || 'en-US,en' }
}
}
]
}
}
const defaultExport = opts => new Plugin(opts)
module.exports = defaultExport

View File

@@ -0,0 +1,324 @@
const test = require('ava')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
// Fixed since 2.1.1?
// test('vanilla: Accept-Language header is missing', async t => {
// const browser = await vanillaPuppeteer.launch({ headless: true })
// const page = await browser.newPage()
// await page.goto('http://httpbin.org/headers')
// const content = await page.content()
// t.true(content.includes(`"User-Agent"`))
// t.false(content.includes(`"Accept-Language"`))
// })
test('vanilla: User-Agent header contains HeadlessChrome', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
await page.goto('http://httpbin.org/headers')
const content = await page.content()
t.true(content.includes(`"User-Agent"`))
t.true(content.includes(`HeadlessChrome`))
})
test('vanilla: navigator.languages is always en-US', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const lang = await page.evaluate(() => navigator.languages)
t.true(lang.length === 1 && lang[0] === 'en-US')
})
test('vanilla: navigator.platform set to host platform', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const platform = await page.evaluate(() => navigator.platform)
switch (process.platform) {
case 'linux':
t.true(platform.includes('Linux')) // TravisCI
break
case 'darwin':
t.true(platform === 'MacIntel')
break
case 'win32':
t.true(platform === 'Win32')
break
default:
t.true(platform === process.platform)
}
})
test('stealth: Accept-Language header with default locale', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
await page.goto('http://httpbin.org/headers')
const content = await page.content()
t.true(content.includes(`"User-Agent"`))
t.true(content.includes(`"Accept-Language": "en-US,en;q=0.9"`))
})
test('stealth: Accept-Language header with optional locale', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(
Plugin({ locale: 'de-DE,de' })
)
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
await page.goto('http://httpbin.org/headers')
const content = await page.content()
t.true(content.includes(`"User-Agent"`))
t.true(content.includes(`"Accept-Language": "de-DE,de;q=0.9"`))
})
test('stealth: User-Agent header does not contain HeadlessChrome', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
await page.goto('http://httpbin.org/headers')
const content = await page.content()
t.true(content.includes(`"User-Agent"`))
t.false(content.includes(`HeadlessChrome`))
})
test('stealth: User-Agent header with custom userAgent', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(
Plugin({ userAgent: 'MyFunkyUA/1.0' })
)
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
await page.goto('http://httpbin.org/headers')
const content = await page.content()
t.true(content.includes(`"User-Agent": "MyFunkyUA/1.0"`))
})
test('stealth: navigator.languages with default locale', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const lang = await page.evaluate(() => navigator.languages)
t.true(lang.length === 2 && lang[0] === 'en-US' && lang[1] === 'en')
})
test('stealth: navigator.languages with custom locale', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(
Plugin({ locale: 'de-DE,de' })
)
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const langs = await page.evaluate(() => navigator.languages)
t.deepEqual(langs, ['de-DE', 'de'])
const lang = await page.evaluate(() => navigator.language)
t.deepEqual(lang, 'de-DE')
})
test('stealth: navigator.platform with maskLinux true (default)', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(
Plugin({
userAgent:
'Mozilla/5.0 (X11; Ubuntu; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.9.9999.99 Safari/537.36'
})
)
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const platform = await page.evaluate(() => navigator.platform)
t.true(platform === 'Win32')
})
test('stealth: navigator.platform with maskLinux false', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(
Plugin({
userAgent:
'Mozilla/5.0 (X11; Ubuntu; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.9.9999.99 Safari/537.36',
maskLinux: false
})
)
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const platform = await page.evaluate(() => navigator.platform)
t.true(platform === 'Linux')
})
const _testUAHint = async (userAgent, locale) => {
const puppeteer = addExtra(vanillaPuppeteer).use(
Plugin({ userAgent, locale })
)
const browser = await puppeteer.launch({
headless: false, // only works on headful
args: ['--enable-features=UserAgentClientHint']
})
const majorVersion = parseInt(
(await browser.version()).match(/\/([^\.]+)/)[1]
)
if (majorVersion < 88) {
return null // Skip test on browsers that don't support UA hints
}
const page = await browser.newPage()
await page.goto('https://headers.cf/headers/?format=raw')
return page
}
test('stealth: test if UA hints are correctly set - Windows 10', async t => {
const page = await _testUAHint(
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.9999.99 Safari/537.36',
'en-AU'
)
if (!page) {
t.true(true) // skip
return
}
const firstLoad = await page.content()
t.true(
firstLoad.includes(
`sec-ch-ua: "Google Chrome";v="99", " Not;A Brand";v="99", "Chromium";v="99"`
)
)
t.true(firstLoad.includes(`Accept-Language: en-AU`))
await page.reload()
const secondLoad = await page.content()
if (secondLoad.includes('sec-ch-ua-full-version')) {
t.true(secondLoad.includes('sec-ch-ua-mobile: ?0'))
t.true(secondLoad.includes('sec-ch-ua-full-version: "99.0.9999.99"'))
t.true(secondLoad.includes('sec-ch-ua-arch: "x86"'))
t.true(secondLoad.includes('sec-ch-ua-platform: "Windows"'))
t.true(secondLoad.includes('sec-ch-ua-platform-version: "10.0"'))
t.true(secondLoad.includes('sec-ch-ua-model: ""'))
}
})
test('stealth: test if UA hints are correctly set - macOS 11', async t => {
const page = await _testUAHint(
'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.9999.99 Safari/537.36',
'de-DE'
)
if (!page) {
t.true(true) // skip
return
}
const firstLoad = await page.content()
t.true(
firstLoad.includes(
`sec-ch-ua: "Google Chrome";v="99", " Not;A Brand";v="99", "Chromium";v="99"`
)
)
t.true(firstLoad.includes(`Accept-Language: de-DE`))
await page.reload()
const secondLoad = await page.content()
if (secondLoad.includes('sec-ch-ua-full-version')) {
t.true(secondLoad.includes('sec-ch-ua-mobile: ?0'))
t.true(secondLoad.includes('sec-ch-ua-full-version: "99.0.9999.99"'))
t.true(secondLoad.includes('sec-ch-ua-arch: "x86"'))
t.true(secondLoad.includes('sec-ch-ua-platform: "Mac OS X"'))
t.true(secondLoad.includes('sec-ch-ua-platform-version: "11_1_0"'))
t.true(secondLoad.includes('sec-ch-ua-model: ""'))
}
})
test('stealth: test if UA hints are correctly set - Android 10', async t => {
const page = await _testUAHint(
'Mozilla/5.0 (Linux; Android 10; SM-P205) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.9999.99 Safari/537.36',
'nl-NL'
)
if (!page) {
t.true(true) // skip
return
}
const firstLoad = await page.content()
t.true(
firstLoad.includes(
`sec-ch-ua: "Google Chrome";v="99", " Not;A Brand";v="99", "Chromium";v="99"`
)
)
t.true(firstLoad.includes(`Accept-Language: nl-NL`))
await page.reload()
const secondLoad = await page.content()
if (secondLoad.includes('sec-ch-ua-full-version')) {
t.true(secondLoad.includes('sec-ch-ua-mobile: ?1'))
t.true(secondLoad.includes('sec-ch-ua-full-version: "99.0.9999.99"'))
t.true(secondLoad.includes('sec-ch-ua-arch: ""'))
t.true(secondLoad.includes('sec-ch-ua-platform: "Android"'))
t.true(secondLoad.includes('sec-ch-ua-platform-version: "10"'))
t.true(secondLoad.includes('sec-ch-ua-model: "SM-P205"'))
}
})
async function userAgentData() {
if (!('userAgentData' in navigator)) {
return undefined
}
// https://wicg.github.io/ua-client-hints/#getHighEntropyValues
const UADataProps = ['brands', 'mobile']
const UADataValues = [
'architecture', // "arm"
'bitness', // "64"
'model', // "X644GTM"
'platform', // "PhoneOS"
'platformVersion', // "10A"
'uaFullVersion' // "73.32.AGX.5"
]
const highEntropy = await navigator.userAgentData.getHighEntropyValues(
UADataValues
)
const result = {
...highEntropy,
...Object.fromEntries(UADataProps.map(k => [k, navigator.userAgentData[k]]))
}
return result
}
test('stealth: test if UA hints are correctly set - Windows 10 Generic', async t => {
const userAgent =
'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.9999.99 Safari/537.36'
const locale = 'en-AU'
const puppeteer = addExtra(vanillaPuppeteer).use(
Plugin({
userAgent,
locale
})
)
const browser = await puppeteer.launch({
headless: true
})
const majorVersion = parseInt(
(await browser.version()).match(/\/([^\.]+)/)[1]
)
if (majorVersion < 90) {
t.truthy('foo')
console.log('Skipping test, browser version too old', majorVersion)
return
}
const page = await browser.newPage()
await page.goto('https://example.com') // secure context
const results = await page.evaluate(userAgentData)
t.is(results.platform, 'Windows')
t.is(results.platformVersion, '10.0')
t.is(results.uaFullVersion, '99.0.9999.99')
const language = await page.evaluate(() => navigator.language)
t.is(language, locale)
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,54 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/ab0047d1af7dc38412744abdb61bcfc35c42dc34/packages/puppeteer-extra-plugin-stealth/evasions/user-agent-override/index.js#L42-L203)
- `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`)
- `opts.userAgent` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The user agent to use (default: browser.userAgent())
- `opts.locale` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The locale to use in `Accept-Language` header and in `navigator.languages` (default: `en-US,en`)
- `opts.maskLinux` **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)?** Wether to hide Linux as platform in the user agent or not - true by default
**Extends: PuppeteerExtraPlugin**
Fixes the UserAgent info (composed of UA string, Accept-Language, Platform, and UA hints).
If you don't provide any values this plugin will default to using the regular UserAgent string (while stripping the headless part).
Default language is set to "en-US,en", the other settings match the UserAgent string.
If you are running on Linux, it will mask the settins to look like Windows. This behavior can be disabled with the `maskLinux` option.
By default puppeteer will not set a `Accept-Language` header in headless:
It's (theoretically) possible to fix that using either `page.setExtraHTTPHeaders` or a `--lang` launch arg.
Unfortunately `page.setExtraHTTPHeaders` will lowercase everything and launch args are not always available. :)
In addition, the `navigator.platform` property is always set to the host value, e.g. `Linux` which makes detection very easy.
Note: You cannot use the regular `page.setUserAgent()` puppeteer call in your code,
as it will reset the language and platform values you set with this plugin.
Example:
```javascript
const puppeteer = require('puppeteer-extra')
const StealthPlugin = require('puppeteer-extra-plugin-stealth')
const stealth = StealthPlugin()
// Remove this specific stealth plugin from the default set
stealth.enabledEvasions.delete('user-agent-override')
puppeteer.use(stealth)
// Stealth plugins are just regular `puppeteer-extra` plugins and can be added as such
const UserAgentOverride = require('puppeteer-extra-plugin-stealth/evasions/user-agent-override')
// Define custom UA and locale
const ua = UserAgentOverride({
userAgent: 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)',
locale: 'de-DE,de'
})
puppeteer.use(ua)
```
---

View File

@@ -0,0 +1,59 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const withUtils = require('../_utils/withUtils')
/**
* Fix WebGL Vendor/Renderer being set to Google in headless mode
*
* Example data (Apple Retina MBP 13): {vendor: "Intel Inc.", renderer: "Intel(R) Iris(TM) Graphics 6100"}
*
* @param {Object} [opts] - Options
* @param {string} [opts.vendor] - The vendor string to use (default: `Intel Inc.`)
* @param {string} [opts.renderer] - The renderer string (default: `Intel Iris OpenGL Engine`)
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/webgl.vendor'
}
/* global WebGLRenderingContext WebGL2RenderingContext */
async onPageCreated(page) {
await withUtils(page).evaluateOnNewDocument((utils, opts) => {
const getParameterProxyHandler = {
apply: function(target, ctx, args) {
const param = (args || [])[0]
const result = utils.cache.Reflect.apply(target, ctx, args)
// UNMASKED_VENDOR_WEBGL
if (param === 37445) {
return opts.vendor || 'Intel Inc.' // default in headless: Google Inc.
}
// UNMASKED_RENDERER_WEBGL
if (param === 37446) {
return opts.renderer || 'Intel Iris OpenGL Engine' // default in headless: Google SwiftShader
}
return result
}
}
// There's more than one WebGL rendering context
// https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext#Browser_compatibility
// To find out the original values here: Object.getOwnPropertyDescriptors(WebGLRenderingContext.prototype.getParameter)
const addProxy = (obj, propName) => {
utils.replaceWithProxy(obj, propName, getParameterProxyHandler)
}
// For whatever weird reason loops don't play nice with Object.defineProperty, here's the next best thing:
addProxy(WebGLRenderingContext.prototype, 'getParameter')
addProxy(WebGL2RenderingContext.prototype, 'getParameter')
}, this.opts)
}
}
module.exports = function(pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,222 @@
const test = require('ava')
const {
getVanillaFingerPrint,
getStealthFingerPrint
} = require('../../test/util')
const { vanillaPuppeteer, addExtra } = require('../../test/util')
const Plugin = require('.')
const { errors } = require('puppeteer')
// FIXME: This changed in more recent chrome versions
// test('vanilla: videoCard is Google Inc', async t => {
// const pageFn = async page => await page.evaluate(() => window.chrome) // eslint-disable-line
// const { videoCard } = await getVanillaFingerPrint(pageFn)
// t.deepEqual(videoCard, ['Google Inc.', 'Google SwiftShader'])
// })
test('stealth: videoCard is Intel Inc', async t => {
const pageFn = async page => await page.evaluate(() => window.chrome) // eslint-disable-line
const { videoCard } = await getStealthFingerPrint(Plugin, pageFn)
t.deepEqual(videoCard, ['Intel Inc.', 'Intel Iris OpenGL Engine'])
})
test('stealth: customized values', async t => {
const pageFn = async page => await page.evaluate(() => window.chrome) // eslint-disable-line
const { videoCard } = await getStealthFingerPrint(Plugin, pageFn, {
vendor: 'foo',
renderer: 'bar'
})
t.deepEqual(videoCard, ['foo', 'bar'])
})
/* global WebGLRenderingContext */
async function extendedTests() {
const results = {}
async function test(name, fn) {
const detectionPassed = await fn()
if (detectionPassed) console.log(`Chrome headless detected via ${name}`)
results[name] = detectionPassed
}
const canvas = document.createElement('canvas')
const context = canvas.getContext('webgl')
await test('descriptorsOK', _ => {
const descriptors = Object.getOwnPropertyDescriptors(
WebGLRenderingContext.prototype
)
const str = descriptors.getParameter.toString()
return str === `[object Object]`
})
await test('toStringOK', _ => {
const str = context.getParameter.toString()
return str === `function getParameter() { [native code] }`
})
await test('toStringOK2', _ => {
const str = WebGLRenderingContext.prototype.getParameter.toString()
return str === `function getParameter() { [native code] }`
})
// Make sure we not reveal our proxy through errors
await test('errorOK', _ => {
try {
return context.getParameter()
} catch (err) {
return !err.stack.includes(`at Object.apply`)
}
})
// Should not throw (that was old stealth behavior)
await test('elementOK', _ => {
try {
return context.getParameter(123) === null
} catch (_) {
return false
}
})
return results
}
test('vanilla: webgl is native', async t => {
const pageFn = async page => {
// page.on('console', msg => {
// console.log('Page console: ', msg.text())
// })
return await page.evaluate(extendedTests) // eslint-disable-line
}
const { pageFnResult: result } = await getVanillaFingerPrint(pageFn)
const wasHeadlessDetected = Object.values(result).some(e => e === false)
if (wasHeadlessDetected) {
console.log(result)
}
t.false(wasHeadlessDetected)
})
test('stealth: webgl is native', async t => {
const pageFn = async page => await page.evaluate(extendedTests) // eslint-disable-line
const { pageFnResult: result } = await getStealthFingerPrint(Plugin, pageFn)
const wasHeadlessDetected = Object.values(result).some(e => e === false)
if (wasHeadlessDetected) {
console.log(result)
}
t.false(wasHeadlessDetected)
})
/**
* A very simple method to retrieve the name of the default videocard of the system
* using webgl.
*
* Example (Apple Retina MBP 13): {vendor: "Intel Inc.", renderer: "Intel(R) Iris(TM) Graphics 6100"}
*
* @see https://stackoverflow.com/questions/49267764/how-to-get-the-video-card-driver-name-using-javascript-browser-side
* @returns {Object}
*/
function getVideoCardInfo(context = 'webgl') {
const gl = document.createElement('canvas').getContext(context)
if (!gl) {
return {
error: 'no webgl'
}
}
const debugInfo = gl.getExtension('WEBGL_debug_renderer_info')
if (debugInfo) {
return {
vendor: gl.getParameter(debugInfo.UNMASKED_VENDOR_WEBGL),
renderer: gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL)
}
}
return {
error: 'no WEBGL_debug_renderer_info'
}
}
test('stealth: handles WebGLRenderingContext', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const videoCardInfo = await page.evaluate(getVideoCardInfo, 'webgl')
t.is(videoCardInfo.error, undefined)
t.is(videoCardInfo.vendor, 'Intel Inc.')
t.is(videoCardInfo.renderer, 'Intel Iris OpenGL Engine')
})
test('stealth: handles WebGL2RenderingContext', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const videoCardInfo = await page.evaluate(getVideoCardInfo, 'webgl2')
t.is(videoCardInfo.error, undefined)
t.is(videoCardInfo.vendor, 'Intel Inc.')
t.is(videoCardInfo.renderer, 'Intel Iris OpenGL Engine')
})
test('vanilla: normal toString stuff', async t => {
const browser = await vanillaPuppeteer.launch({ headless: true })
const page = await browser.newPage()
const test1 = await page.evaluate(() => {
return WebGLRenderingContext.prototype.getParameter.toString + ''
})
t.is(test1, 'function toString() { [native code] }')
const test2 = await page.evaluate(() => {
return WebGLRenderingContext.prototype.getParameter.toString()
})
t.is(test2, 'function getParameter() { [native code] }')
})
test('stealth: will not leak toString stuff', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const test1 = await page.evaluate(() => {
return WebGLRenderingContext.prototype.getParameter.toString + ''
})
t.is(test1, 'function toString() { [native code] }') // returns function () { [native code] }
const test2 = await page.evaluate(() => {
return WebGLRenderingContext.prototype.getParameter.toString()
})
t.is(test2, 'function getParameter() { [native code] }')
})
test('stealth: sets user opts correctly', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(
Plugin({ vendor: 'alice', renderer: 'bob' })
)
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const videoCardInfo = await page.evaluate(getVideoCardInfo, 'webgl')
t.is(videoCardInfo.error, undefined)
t.is(videoCardInfo.vendor, 'alice')
t.is(videoCardInfo.renderer, 'bob')
})
test('stealth: does not affect protoype', async t => {
const puppeteer = addExtra(vanillaPuppeteer).use(
Plugin({ vendor: 'alice', renderer: 'bob' })
)
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
const result = await page.evaluate(() => {
try {
return WebGLRenderingContext.prototype.getParameter(37445)
} catch (err) {
return err.message
}
})
t.is(result, 'Illegal invocation')
})

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,21 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/webgl.vendor/index.js#L17-L55)
- `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`)
- `opts.vendor` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The vendor string to use (default: `Intel Inc.`)
- `opts.renderer` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The renderer string (default: `Intel Iris OpenGL Engine`)
**Extends: PuppeteerExtraPlugin**
Fix WebGL Vendor/Renderer being set to Google in headless mode
Example data (Apple Retina MBP 13): {vendor: "Intel Inc.", renderer: "Intel(R) Iris(TM) Graphics 6100"}
---

View File

@@ -0,0 +1,44 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
/**
* Fix missing window.outerWidth/window.outerHeight in headless mode
* Will also set the viewport to match window size, unless specified by user
*/
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth/evasions/window.outerdimensions'
}
async onPageCreated(page) {
// Chrome returns undefined, Firefox false
await page.evaluateOnNewDocument(() => {
try {
if (window.outerWidth && window.outerHeight) {
return // nothing to do here
}
const windowFrame = 85 // probably OS and WM dependent
window.outerWidth = window.innerWidth
window.outerHeight = window.innerHeight + windowFrame
} catch (err) {}
})
}
async beforeLaunch(options) {
// Have viewport match window size, unless specified by user
// https://github.com/GoogleChrome/puppeteer/issues/3688
if (!('defaultViewport' in options)) {
options.defaultViewport = null
}
return options
}
}
module.exports = function(pluginConfig) {
return new Plugin(pluginConfig)
}

View File

@@ -0,0 +1,4 @@
{
"private": true,
"main": "index.js"
}

View File

@@ -0,0 +1,18 @@
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: Plugin](#class-plugin)
### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/window.outerdimensions/index.js#L9-L40)
- `opts` (optional, default `{}`)
**Extends: PuppeteerExtraPlugin**
Fix missing window.outerWidth/window.outerHeight in headless mode
Will also set the viewport to match window size, unless specified by user
---

View File

@@ -0,0 +1,97 @@
'use strict'
// taken from: https://github.com/paulirish/headless-cat-n-mouse/blob/master/detect-headless.js
// initial detects from @antoinevastel
// http://antoinevastel.github.io/bot%20detection/2018/01/17/detect-chrome-headless-v2.html
module.exports = async function() {
const results = {}
async function test(name, fn) {
const detectionPassed = await fn()
if (detectionPassed) {
console.log(`WARNING: Chrome headless detected via ${name}`)
} else {
console.log(`PASS: Chrome headless NOT detected via ${name}`)
}
results[name] = detectionPassed
}
await test('userAgent', _ => {
return /HeadlessChrome/.test(window.navigator.userAgent)
})
// Detects the --enable-automation || --headless flags
// Will return true in headful if --enable-automation is provided
await test('navigator.webdriver present', _ => {
return 'webdriver' in navigator
})
await test('window.chrome missing', _ => {
return /Chrome/.test(window.navigator.userAgent) && !window.chrome
})
await test('permissions API', async _ => {
const permissionStatus = await navigator.permissions.query({
name: 'notifications'
})
// eslint-disable-next-line
return (
Notification.permission === 'denied' && // eslint-disable-line no-undef
permissionStatus.state === 'prompt'
)
})
await test('permissions API overriden', _ => {
const permissions = window.navigator.permissions
if (permissions.query.toString() !== 'function query() { [native code] }')
return true
if (
permissions.query.toString.toString() !==
'function toString() { [native code] }'
)
return true
if (
permissions.query.toString.hasOwnProperty('[[Handler]]') && // eslint-disable-line no-prototype-builtins
permissions.query.toString.hasOwnProperty('[[Target]]') && // eslint-disable-line no-prototype-builtins
permissions.query.toString.hasOwnProperty('[[IsRevoked]]') // eslint-disable-line no-prototype-builtins
)
return true
if (permissions.hasOwnProperty('query')) return true // eslint-disable-line no-prototype-builtins
})
await test('navigator.plugins empty', _ => {
return navigator.plugins.length === 0
})
await test('navigator.languages blank', _ => {
return navigator.languages === ''
})
await test('iFrame for fresh window object', _ => {
// evaluateOnNewDocument scripts don't apply within [srcdoc] (or [sandbox]) iframes
// https://github.com/GoogleChrome/puppeteer/issues/1106#issuecomment-359313898
const iframe = document.createElement('iframe')
iframe.srcdoc = 'page intentionally left blank'
document.body.appendChild(iframe)
// Here we would need to rerun all tests with `iframe.contentWindow` as `window`
// Example:
return iframe.contentWindow.navigator.plugins.length === 0
})
// This detects that a devtools protocol agent is attached.
// So it will also pass true in headful Chrome if the devtools window is attached
await test('toString', _ => {
let gotYou = 0
const spooky = /./
spooky.toString = function() {
gotYou++
return 'spooky'
}
console.debug(spooky)
return gotYou > 1
})
return results
}

View File

@@ -0,0 +1,23 @@
'use strict'
const puppeteer = require('puppeteer-extra')
puppeteer.use(require('puppeteer-extra-plugin-stealth')())
const detectHeadless = require('./detect-headless')
;(async () => {
const browser = await puppeteer.launch({ args: ['--no-sandbox'] })
const page = await browser.newPage()
page.on('console', msg => {
console.log('Page console: ', msg.text())
})
await page.goto('about:blank')
const detectionResults = await page.evaluate(detectHeadless)
console.assert(
Object.keys(detectionResults).length,
'No detection results returned.'
)
await browser.close()
})()

View File

@@ -0,0 +1,26 @@
'use strict'
const puppeteer = require('puppeteer-extra')
// Enable stealth plugin
puppeteer.use(require('puppeteer-extra-plugin-stealth')())
;(async () => {
// Launch the browser in headless mode and set up a page.
const browser = await puppeteer.launch({
args: ['--no-sandbox'],
headless: true
})
const page = await browser.newPage()
// Navigate to the page that will perform the tests.
const testUrl =
'https://intoli.com/blog/' +
'not-possible-to-block-chrome-headless/chrome-headless-test.html'
await page.goto(testUrl)
// Save a screenshot of the results.
const screenshotPath = '/tmp/headless-test-result.png'
await page.screenshot({ path: screenshotPath })
console.log('have a look at the screenshot:', screenshotPath)
await browser.close()
})()

111
node_modules/puppeteer-extra-plugin-stealth/index.d.ts generated vendored Normal file
View File

@@ -0,0 +1,111 @@
export = defaultExport;
declare function defaultExport(opts?: {
enabledEvasions?: Set<string>;
}): StealthPlugin;
declare const StealthPlugin_base: typeof import("puppeteer-extra-plugin").PuppeteerExtraPlugin;
/**
* Stealth mode: Applies various techniques to make detection of headless puppeteer harder. 💯
*
* ### Purpose
* There are a couple of ways the use of puppeteer can easily be detected by a target website.
* The addition of `HeadlessChrome` to the user-agent being only the most obvious one.
*
* The goal of this plugin is to be the definite companion to puppeteer to avoid
* detection, applying new techniques as they surface.
*
* As this cat & mouse game is in it's infancy and fast-paced the plugin
* is kept as flexibile as possible, to support quick testing and iterations.
*
* ### Modularity
* This plugin uses `puppeteer-extra`'s dependency system to only require
* code mods for evasions that have been enabled, to keep things modular and efficient.
*
* The `stealth` plugin is a convenience wrapper that requires multiple [evasion techniques](./evasions/)
* automatically and comes with defaults. You could also bypass the main module and require
* specific evasion plugins yourself, if you whish to do so (as they're standalone `puppeteer-extra` plugins):
*
* ```es6
* // bypass main module and require a specific stealth plugin directly:
* puppeteer.use(require('puppeteer-extra-plugin-stealth/evasions/console.debug')())
* ```
*
* ### Contributing
* PRs are welcome, if you want to add a new evasion technique I suggest you
* look at the [template](./evasions/_template) to kickstart things.
*
* ### Kudos
* Thanks to [Evan Sangaline](https://intoli.com/blog/not-possible-to-block-chrome-headless/) and [Paul Irish](https://github.com/paulirish/headless-cat-n-mouse) for kickstarting the discussion!
*
* ---
*
* @todo
* - white-/blacklist with url globs (make this a generic plugin method?)
* - dynamic whitelist based on function evaluation
*
* @example
* const puppeteer = require('puppeteer-extra')
* // Enable stealth plugin with all evasions
* puppeteer.use(require('puppeteer-extra-plugin-stealth')())
*
*
* ;(async () => {
* // Launch the browser in headless mode and set up a page.
* const browser = await puppeteer.launch({ args: ['--no-sandbox'], headless: true })
* const page = await browser.newPage()
*
* // Navigate to the page that will perform the tests.
* const testUrl = 'https://intoli.com/blog/' +
* 'not-possible-to-block-chrome-headless/chrome-headless-test.html'
* await page.goto(testUrl)
*
* // Save a screenshot of the results.
* const screenshotPath = '/tmp/headless-test-result.png'
* await page.screenshot({path: screenshotPath})
* console.log('have a look at the screenshot:', screenshotPath)
*
* await browser.close()
* })()
*
* @param {Object} [opts] - Options
* @param {Set<string>} [opts.enabledEvasions] - Specify which evasions to use (by default all)
*
*/
declare class StealthPlugin extends StealthPlugin_base {
constructor(opts?: {});
get defaults(): {
availableEvasions: Set<string>;
enabledEvasions: Set<any>;
};
/**
* Get all available evasions.
*
* Please look into the [evasions directory](./evasions/) for an up to date list.
*
* @type {Set<string>} - A Set of all available evasions.
*
* @example
* const pluginStealth = require('puppeteer-extra-plugin-stealth')()
* console.log(pluginStealth.availableEvasions) // => Set { 'user-agent', 'console.debug' }
* puppeteer.use(pluginStealth)
*/
get availableEvasions(): Set<string>;
/**
* @private
*/
set enabledEvasions(arg: Set<string>);
/**
* Get all enabled evasions.
*
* Enabled evasions can be configured either through `opts` or by modifying this property.
*
* @type {Set<string>} - A Set of all enabled evasions.
*
* @example
* // Remove specific evasion from enabled ones dynamically
* const pluginStealth = require('puppeteer-extra-plugin-stealth')()
* pluginStealth.enabledEvasions.delete('console.debug')
* puppeteer.use(pluginStealth)
*/
get enabledEvasions(): Set<string>;
onBrowser(browser: any): Promise<void>;
}

177
node_modules/puppeteer-extra-plugin-stealth/index.js generated vendored Normal file
View File

@@ -0,0 +1,177 @@
'use strict'
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
/**
* Stealth mode: Applies various techniques to make detection of headless puppeteer harder. 💯
*
* ### Purpose
* There are a couple of ways the use of puppeteer can easily be detected by a target website.
* The addition of `HeadlessChrome` to the user-agent being only the most obvious one.
*
* The goal of this plugin is to be the definite companion to puppeteer to avoid
* detection, applying new techniques as they surface.
*
* As this cat & mouse game is in it's infancy and fast-paced the plugin
* is kept as flexibile as possible, to support quick testing and iterations.
*
* ### Modularity
* This plugin uses `puppeteer-extra`'s dependency system to only require
* code mods for evasions that have been enabled, to keep things modular and efficient.
*
* The `stealth` plugin is a convenience wrapper that requires multiple [evasion techniques](./evasions/)
* automatically and comes with defaults. You could also bypass the main module and require
* specific evasion plugins yourself, if you whish to do so (as they're standalone `puppeteer-extra` plugins):
*
* ```es6
* // bypass main module and require a specific stealth plugin directly:
* puppeteer.use(require('puppeteer-extra-plugin-stealth/evasions/console.debug')())
* ```
*
* ### Contributing
* PRs are welcome, if you want to add a new evasion technique I suggest you
* look at the [template](./evasions/_template) to kickstart things.
*
* ### Kudos
* Thanks to [Evan Sangaline](https://intoli.com/blog/not-possible-to-block-chrome-headless/) and [Paul Irish](https://github.com/paulirish/headless-cat-n-mouse) for kickstarting the discussion!
*
* ---
*
* @todo
* - white-/blacklist with url globs (make this a generic plugin method?)
* - dynamic whitelist based on function evaluation
*
* @example
* const puppeteer = require('puppeteer-extra')
* // Enable stealth plugin with all evasions
* puppeteer.use(require('puppeteer-extra-plugin-stealth')())
*
*
* ;(async () => {
* // Launch the browser in headless mode and set up a page.
* const browser = await puppeteer.launch({ args: ['--no-sandbox'], headless: true })
* const page = await browser.newPage()
*
* // Navigate to the page that will perform the tests.
* const testUrl = 'https://intoli.com/blog/' +
* 'not-possible-to-block-chrome-headless/chrome-headless-test.html'
* await page.goto(testUrl)
*
* // Save a screenshot of the results.
* const screenshotPath = '/tmp/headless-test-result.png'
* await page.screenshot({path: screenshotPath})
* console.log('have a look at the screenshot:', screenshotPath)
*
* await browser.close()
* })()
*
* @param {Object} [opts] - Options
* @param {Set<string>} [opts.enabledEvasions] - Specify which evasions to use (by default all)
*
*/
class StealthPlugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return 'stealth'
}
get defaults() {
const availableEvasions = new Set([
'chrome.app',
'chrome.csi',
'chrome.loadTimes',
'chrome.runtime',
'defaultArgs',
'iframe.contentWindow',
'media.codecs',
'navigator.hardwareConcurrency',
'navigator.languages',
'navigator.permissions',
'navigator.plugins',
'navigator.webdriver',
'sourceurl',
'user-agent-override',
'webgl.vendor',
'window.outerdimensions'
])
return {
availableEvasions,
// Enable all available evasions by default
enabledEvasions: new Set([...availableEvasions])
}
}
/**
* Requires evasion techniques dynamically based on configuration.
*
* @private
*/
get dependencies() {
return new Set(
[...this.opts.enabledEvasions].map(e => `${this.name}/evasions/${e}`)
)
}
/**
* Get all available evasions.
*
* Please look into the [evasions directory](./evasions/) for an up to date list.
*
* @type {Set<string>} - A Set of all available evasions.
*
* @example
* const pluginStealth = require('puppeteer-extra-plugin-stealth')()
* console.log(pluginStealth.availableEvasions) // => Set { 'user-agent', 'console.debug' }
* puppeteer.use(pluginStealth)
*/
get availableEvasions() {
return this.defaults.availableEvasions
}
/**
* Get all enabled evasions.
*
* Enabled evasions can be configured either through `opts` or by modifying this property.
*
* @type {Set<string>} - A Set of all enabled evasions.
*
* @example
* // Remove specific evasion from enabled ones dynamically
* const pluginStealth = require('puppeteer-extra-plugin-stealth')()
* pluginStealth.enabledEvasions.delete('console.debug')
* puppeteer.use(pluginStealth)
*/
get enabledEvasions() {
return this.opts.enabledEvasions
}
/**
* @private
*/
set enabledEvasions(evasions) {
this.opts.enabledEvasions = evasions
}
async onBrowser(browser) {
if (browser && browser.setMaxListeners) {
// Increase event emitter listeners to prevent MaxListenersExceededWarning
browser.setMaxListeners(30)
}
}
}
/**
* Default export, PuppeteerExtraStealthPlugin
*
* @param {Object} [opts] - Options
* @param {Set<string>} [opts.enabledEvasions] - Specify which evasions to use (by default all)
*/
const defaultExport = opts => new StealthPlugin(opts)
module.exports = defaultExport
// const moduleExport = defaultExport
// moduleExport.StealthPlugin = StealthPlugin
// module.exports = moduleExport

View File

@@ -0,0 +1,54 @@
'use strict'
const PLUGIN_NAME = 'stealth'
const test = require('ava')
const Plugin = require('.')
test('is a function', async t => {
t.is(typeof Plugin, 'function')
})
test('should have the basic class members', async t => {
const instance = Plugin()
t.is(instance.name, PLUGIN_NAME)
t.true(instance._isPuppeteerExtraPlugin)
})
test('should have the public child class members', async t => {
const instance = Plugin()
const prototype = Object.getPrototypeOf(instance)
const childClassMembers = Object.getOwnPropertyNames(prototype)
t.true(childClassMembers.includes('constructor'))
t.true(childClassMembers.includes('name'))
t.true(childClassMembers.includes('name'))
t.true(childClassMembers.includes('defaults'))
t.true(childClassMembers.includes('availableEvasions'))
t.true(childClassMembers.includes('enabledEvasions'))
t.is(childClassMembers.length, 7)
})
test('should have opts with default values', async t => {
const instance = Plugin()
t.deepEqual(instance.opts.enabledEvasions, instance.availableEvasions)
})
test('should add all dependencies dynamically', async t => {
const instance = Plugin()
const deps = new Set(
[...instance.opts.enabledEvasions].map(e => `${PLUGIN_NAME}/evasions/${e}`)
)
t.deepEqual(instance.dependencies, deps)
})
test('should add all dependencies dynamically including changes', async t => {
const instance = Plugin()
const fakeDep = 'foobar'
instance.enabledEvasions = new Set([fakeDep])
t.deepEqual(
instance.dependencies,
new Set([`${PLUGIN_NAME}/evasions/${fakeDep}`])
)
})

View File

@@ -0,0 +1,71 @@
{
"name": "puppeteer-extra-plugin-stealth",
"version": "2.11.2",
"description": "Stealth mode: Applies various techniques to make detection of headless puppeteer harder.",
"main": "index.js",
"typings": "index.d.ts",
"repository": "berstend/puppeteer-extra",
"homepage": "https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth#readme",
"author": "berstend",
"license": "MIT",
"scripts": {
"docs": "run-s docs-for-plugin postdocs-for-plugin docs-for-evasions postdocs-for-evasions types",
"docs-for-plugin": "documentation readme --quiet --shallow --github --markdown-theme transitivebs --readme-file readme.md --section API index.js",
"postdocs-for-plugin": "npx prettier --write readme.md",
"docs-for-evasions": "cd ./evasions && loop \"documentation readme --quiet --shallow --github --markdown-theme transitivebs --readme-file readme.md --section API index.js\"",
"postdocs-for-evasions": "cd ./evasions && loop \"npx prettier --write readme.md\"",
"lint": "eslint --ext .js .",
"test:js": "ava --concurrency 2 -v",
"test": "run-p test:js",
"test-ci": "run-s test:js",
"types": "npx --package typescript@3.7 tsc --emitDeclarationOnly --declaration --allowJs index.js"
},
"engines": {
"node": ">=8"
},
"keywords": [
"puppeteer",
"puppeteer-extra",
"puppeteer-extra-plugin",
"stealth",
"stealth-mode",
"detection-evasion",
"crawler",
"chrome",
"headless",
"pupeteer"
],
"ava": {
"files": [
"!test/util.js",
"!test/fixtures/sw.js"
]
},
"devDependencies": {
"ava": "2.4.0",
"documentation-markdown-themes": "^12.1.5",
"fpcollect": "^1.0.4",
"fpscanner": "^0.1.5",
"loop": "^3.0.6",
"npm-run-all": "^4.1.5",
"puppeteer": "9"
},
"dependencies": {
"debug": "^4.1.1",
"puppeteer-extra-plugin": "^3.2.3",
"puppeteer-extra-plugin-user-preferences": "^2.4.1"
},
"peerDependencies": {
"playwright-extra": "*",
"puppeteer-extra": "*"
},
"peerDependenciesMeta": {
"puppeteer-extra": {
"optional": true
},
"playwright-extra": {
"optional": true
}
},
"gitHead": "2f4a357f233b35a7a20f16ce007f5ef3f62765b9"
}

329
node_modules/puppeteer-extra-plugin-stealth/readme.md generated vendored Normal file
View File

@@ -0,0 +1,329 @@
# puppeteer-extra-plugin-stealth [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/berstend/puppeteer-extra/test.yml?branch=master&event=push) [![Discord](https://img.shields.io/discord/737009125862408274)](https://extra.community) [![npm](https://img.shields.io/npm/v/puppeteer-extra-plugin-stealth.svg)](https://www.npmjs.com/package/puppeteer-extra-plugin-stealth)
> A plugin for [puppeteer-extra](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra) and [playwright-extra](https://github.com/berstend/puppeteer-extra/tree/master/packages/playwright-extra) to prevent detection.
<p align="center"><img src="https://i.imgur.com/q2xBjqH.png" /></p>
## Install
```bash
yarn add puppeteer-extra-plugin-stealth
# - or -
npm install puppeteer-extra-plugin-stealth
```
If this is your first [puppeteer-extra](https://github.com/berstend/puppeteer-extra) plugin here's everything you need:
```bash
yarn add puppeteer puppeteer-extra puppeteer-extra-plugin-stealth
# - or -
npm install puppeteer puppeteer-extra puppeteer-extra-plugin-stealth
```
## Usage
```js
// puppeteer-extra is a drop-in replacement for puppeteer,
// it augments the installed puppeteer with plugin functionality
const puppeteer = require('puppeteer-extra')
// add stealth plugin and use defaults (all evasion techniques)
const StealthPlugin = require('puppeteer-extra-plugin-stealth')
puppeteer.use(StealthPlugin())
// puppeteer usage as normal
puppeteer.launch({ headless: true }).then(async browser => {
console.log('Running tests..')
const page = await browser.newPage()
await page.goto('https://bot.sannysoft.com')
await page.waitForTimeout(5000)
await page.screenshot({ path: 'testresult.png', fullPage: true })
await browser.close()
console.log(`All done, check the screenshot. ✨`)
})
```
<details>
<summary><strong>TypeScript usage</strong></summary><br/>
> `puppeteer-extra` and most plugins are written in TS,
> so you get perfect type support out of the box. :)
```ts
import puppeteer from 'puppeteer-extra'
import StealthPlugin from 'puppeteer-extra-plugin-stealth'
puppeteer
.use(StealthPlugin())
.launch({ headless: true })
.then(async browser => {
const page = await browser.newPage()
await page.goto('https://bot.sannysoft.com')
await page.waitForTimeout(5000)
await page.screenshot({ path: 'stealth.png', fullPage: true })
await browser.close()
})
```
> Please check this [wiki](https://github.com/berstend/puppeteer-extra/wiki/TypeScript-usage) entry in case you have TypeScript related import issues.
</details><br>
> Please check out the [main documentation](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra) to learn more about `puppeteer-extra` (Firefox usage, other Plugins, etc).
## Status
-**`puppeteer-extra` with stealth passes all public bot tests.**
Please note: I consider this a friendly competition in a rather interesting cat and mouse game. If the other team (👋) wants to detect headless chromium there are still ways to do that (at least I noticed a few, which I'll tackle in future updates).
It's probably impossible to prevent all ways to detect headless chromium, but it should be possible to make it so difficult that it becomes cost-prohibitive or triggers too many false-positives to be feasible.
If something new comes up or you experience a problem, please do your homework and create a PR in a respectful way (this is Github, not reddit) or I might not be motivated to help. :)
## Changelog
> 🎁 **Note:** Until we've automated changelog updates in markdown files please follow the `#announcements` channel in our [discord server](https://discord.gg/vz7PeKk) for the latest updates and changelog info.
_Older changelog:_
#### `v2.4.7`
- New: `user-agent-override` - Used to set a stealthy UA string, language & platform. This also fixes issues with the prior method of setting the `Accept-Language` header through request interception ([#104](https://github.com/berstend/puppeteer-extra/pull/104), kudos to [@Niek](https://github.com/Niek))
- New: `navigator.vendor` - Makes it possible to optionally override navigator.vendor ([#110](https://github.com/berstend/puppeteer-extra/pull/110), thanks [@Niek](https://github.com/Niek))
- Improved: `navigator.webdriver`: Now uses ES6 Proxies to pass `instanceof` tests ([#117](https://github.com/berstend/puppeteer-extra/pull/117), thanks [@aabbccsmith](https://github.com/aabbccsmith))
- Removed: `user-agent`, `accept-language` (now obsolete)
#### `v2.4.2` / `v2.4.1`
- Improved: `iframe.contentWindow` - We now proxy the original window object and smartly redirect calls that might reveal it's true identity, as opposed to mocking it like peasants :)
- Improved: `accept-language` - More robust and it's now possible to [set a custom locale](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth/evasions/accept-language#readme) if needed.
- ⭐️ Passes the [headless-cat-n-mouse](https://github.com/paulirish/headless-cat-n-mouse) test
#### `v2.4.0`
Let's ring the bell for round 2 in this cat and mouse fight 😄
- New: All evasions now have a specific before and after test to make make this whole topic less voodoo
- New: `media.codecs` - we spoof the presence of proprietary codecs in Chromium now
- New & improved: `iframe.contentWindow` - Found a way to fix `srcdoc` frame based detection without breaking recaptcha inline popup & other iframes (please report any issues)
- New: `accept-language` - Adds a missing `Accept-Language` header in headless (capitalized correctly, `page.setExtraHTTPHeaders` is all lowercase which can be detected)
- Improved: `chrome.runtime` - More extensive mocking of the chrome object
- ⭐️ All [fpscanner](https://antoinevastel.com/bots/) tests are now green, as well as all [intoli](https://bot.sannysoft.com) tests and the [`areyouheadless`](https://arh.antoinevastel.com/bots/areyouheadless) test
<details>
<summary><code>v2.1.2</code></summary><br/>
- Improved: `navigator.plugins` - we fully emulate plugins/mimetypes in headless now 🎉
- New: `webgl.vendor` - is otherwise set to "Google" in headless
- New: `window.outerdimensions` - fix missing window.outerWidth/outerHeight and viewport
- Fixed: `navigator.webdriver` now returns undefined instead of false
</details>
## Test results (red is bad)
#### Vanilla puppeteer <strong>without stealth 😢</strong>
<table class="image">
<tr>
<td><figure class="image"><a href="./stealthtests/_results/headless-chromium-vanilla.js.png"><img src="./stealthtests/_results/_thumbs/headless-chromium-vanilla.js.png"></a><figcaption>Chromium + headless</figcaption></figure></td>
<td><figure class="image"><a href="./stealthtests/_results/headful-chromium-vanilla.js.png"><img src="./stealthtests/_results/_thumbs/headful-chromium-vanilla.js.png"></a><figcaption>Chromium + headful</figcaption></figure></td>
<td><figure class="image"><a href="./stealthtests/_results/headless-chrome-vanilla.js.png"><img src="./stealthtests/_results/_thumbs/headless-chrome-vanilla.js.png"></a><figcaption>Chrome + headless</figcaption></figure></td>
<td><figure class="image"><a href="./stealthtests/_results/headful-chrome-vanilla.js.png"><img src="./stealthtests/_results/_thumbs/headful-chrome-vanilla.js.png"></a><figcaption>Chrome + headful</figcaption></figure></td>
</tr>
</table>
#### Puppeteer <strong>with stealth plugin 💯</strong>
<table class="image">
<tr>
<td><figure class="image"><a href="./stealthtests/_results/headless-chromium-stealth.js.png"><img src="./stealthtests/_results/_thumbs/headless-chromium-stealth.js.png"></a><figcaption>Chromium + headless</figcaption></figure></td>
<td><figure class="image"><a href="./stealthtests/_results/headful-chromium-stealth.js.png"><img src="./stealthtests/_results/_thumbs/headful-chromium-stealth.js.png"></a><figcaption>Chromium + headful</figcaption></figure></td>
<td><figure class="image"><a href="./stealthtests/_results/headless-chrome-stealth.js.png"><img src="./stealthtests/_results/_thumbs/headless-chrome-stealth.js.png"></a><figcaption>Chrome + headless</figcaption></figure></td>
<td><figure class="image"><a href="./stealthtests/_results/headful-chrome-stealth.js.png"><img src="./stealthtests/_results/_thumbs/headful-chrome-stealth.js.png"></a><figcaption>Chrome + headful</figcaption></figure></td>
</tr>
</table>
> Note: The `MQ_SCREEN` test is broken on their page (will fail in regular Chrome as well).
Tests have been done using [this test site](https://bot.sannysoft.com/) and [these scripts](./stealthtests/).
#### Improved reCAPTCHA v3 scores
Using stealth also seems to help with maintaining a normal [reCAPTCHA v3 score](https://developers.google.com/recaptcha/docs/v3#score).
<table class="image">
<tr>
<td><figure class="image"><figcaption><code>Regular Puppeteer</code></figcaption><br/><img src="https://i.imgur.com/rHEH69b.png"></figure></td>
<td><figure class="image"><figcaption><code>Stealth Puppeteer</code></figcaption><br/><img src="https://i.imgur.com/2if496Z.png"></figure></td>
</tr>
</table>
Note: The [official test](https://recaptcha-demo.appspot.com/recaptcha-v3-request-scores.php) is to be taken with a grain of salt, as the score is calculated individually per site and multiple other factors (past behaviour, IP address, etc). Based on anecdotal observations it still seems to work as a rough indicator.
_**Tip:** Have a look at the [recaptcha plugin](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-recaptcha) if you have issues with reCAPTCHAs._
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [puppeteer-extra-plugin-stealth \[ ](#puppeteer-extra-plugin-stealth---)
- [Install](#install)
- [Usage](#usage)
- [Status](#status)
- [Changelog](#changelog)
- [`v2.4.7`](#v247)
- [`v2.4.2` / `v2.4.1`](#v242--v241)
- [`v2.4.0`](#v240)
- [Test results (red is bad)](#test-results-red-is-bad)
- [Vanilla puppeteer without stealth 😢](#vanilla-puppeteer-without-stealth-)
- [Puppeteer with stealth plugin 💯](#puppeteer-with-stealth-plugin-)
- [Improved reCAPTCHA v3 scores](#improved-recaptcha-v3-scores)
- [API](#api)
- [Table of Contents](#table-of-contents)
- [class: StealthPlugin](#class-stealthplugin)
- [Purpose](#purpose)
- [Modularity](#modularity)
- [Contributing](#contributing)
- [Kudos](#kudos)
- [.availableEvasions](#availableevasions)
- [.enabledEvasions](#enabledevasions)
- [defaultExport(opts?)](#defaultexportopts)
- [License](#license)
### class: [StealthPlugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/index.js#L72-L162)
- `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`)
- `opts.enabledEvasions` **[Set](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Set)&lt;[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>?** Specify which evasions to use (by default all)
**Extends: PuppeteerExtraPlugin**
Stealth mode: Applies various techniques to make detection of headless puppeteer harder. 💯
#### Purpose
There are a couple of ways the use of puppeteer can easily be detected by a target website.
The addition of `HeadlessChrome` to the user-agent being only the most obvious one.
The goal of this plugin is to be the definite companion to puppeteer to avoid
detection, applying new techniques as they surface.
As this cat & mouse game is in it's infancy and fast-paced the plugin
is kept as flexibile as possible, to support quick testing and iterations.
#### Modularity
This plugin uses `puppeteer-extra`'s dependency system to only require
code mods for evasions that have been enabled, to keep things modular and efficient.
The `stealth` plugin is a convenience wrapper that requires multiple [evasion techniques](./evasions/)
automatically and comes with defaults. You could also bypass the main module and require
specific evasion plugins yourself, if you whish to do so (as they're standalone `puppeteer-extra` plugins):
```es6
// bypass main module and require a specific stealth plugin directly:
puppeteer.use(
require('puppeteer-extra-plugin-stealth/evasions/console.debug')()
)
```
#### Contributing
PRs are welcome, if you want to add a new evasion technique I suggest you
look at the [template](./evasions/_template) to kickstart things.
#### Kudos
Thanks to [Evan Sangaline](https://intoli.com/blog/not-possible-to-block-chrome-headless/) and [Paul Irish](https://github.com/paulirish/headless-cat-n-mouse) for kickstarting the discussion!
---
Example:
```javascript
const puppeteer = require('puppeteer-extra')
// Enable stealth plugin with all evasions
puppeteer.use(require('puppeteer-extra-plugin-stealth')())
;(async () => {
// Launch the browser in headless mode and set up a page.
const browser = await puppeteer.launch({
args: ['--no-sandbox'],
headless: true
})
const page = await browser.newPage()
// Navigate to the page that will perform the tests.
const testUrl =
'https://intoli.com/blog/' +
'not-possible-to-block-chrome-headless/chrome-headless-test.html'
await page.goto(testUrl)
// Save a screenshot of the results.
const screenshotPath = '/tmp/headless-test-result.png'
await page.screenshot({ path: screenshotPath })
console.log('have a look at the screenshot:', screenshotPath)
await browser.close()
})()
```
---
#### .[availableEvasions](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/index.js#L128-L130)
Type: **[Set](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Set)&lt;[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>**
Get all available evasions.
Please look into the [evasions directory](./evasions/) for an up to date list.
Example:
```javascript
const pluginStealth = require('puppeteer-extra-plugin-stealth')()
console.log(pluginStealth.availableEvasions) // => Set { 'user-agent', 'console.debug' }
puppeteer.use(pluginStealth)
```
---
#### .[enabledEvasions](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/index.js#L145-L147)
Type: **[Set](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Set)&lt;[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>**
Get all enabled evasions.
Enabled evasions can be configured either through `opts` or by modifying this property.
Example:
```javascript
// Remove specific evasion from enabled ones dynamically
const pluginStealth = require('puppeteer-extra-plugin-stealth')()
pluginStealth.enabledEvasions.delete('console.debug')
puppeteer.use(pluginStealth)
```
---
### [defaultExport(opts?)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/index.js#L170-L170)
- `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options
- `opts.enabledEvasions` **[Set](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Set)&lt;[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>?** Specify which evasions to use (by default all)
Default export, PuppeteerExtraStealthPlugin
---
## License
Copyright © 2018 - 2023, [berstend̡̲̫̹̠̖͚͓̔̄̓̐̄͛̀͘](mailto:github@berstend.com?subject=[GitHub]%20PuppeteerExtra). Released under the MIT License.

View File

@@ -0,0 +1,162 @@
const test = require('ava')
const { vanillaPuppeteer, addExtra, compareLooseVersionStrings } = require('./util')
const Plugin = require('..')
// Fix CI issues with old versions
const isOldPuppeteerVersion = () => {
const version = process.env.PUPPETEER_VERSION
const isOld = version && (version === '1.9.0' || version === '1.6.2')
return isOld
}
/* global HTMLIFrameElement */
/* global Notification */
test('stealth: will pass Paul Irish', async t => {
const browser = await addExtra(vanillaPuppeteer)
.use(Plugin())
.launch({ headless: true })
const page = await browser.newPage()
await page.exposeFunction('compareLooseVersionStrings', compareLooseVersionStrings)
const detectionResults = await page.evaluate(detectHeadless)
await browser.close()
if (isOldPuppeteerVersion()) {
t.true(true)
return
}
const wasHeadlessDetected = Object.values(detectionResults).some(Boolean)
if (wasHeadlessDetected) {
console.log(detectionResults)
}
t.false(wasHeadlessDetected)
})
async function detectHeadless() {
const results = {}
async function test(name, fn) {
const detectionPassed = await fn()
if (detectionPassed) console.log(`Chrome headless detected via ${name}`)
results[name] = detectionPassed
}
await test('userAgent', _ => {
return /HeadlessChrome/.test(window.navigator.userAgent)
})
// navigator.webdriver behavior change since release 89.0.4339.0. See also #448
if (await compareLooseVersionStrings(navigator.userAgent, '89.0.4339.0') >= 0) {
await test('navigator.webdriver is not false', _ => {
return navigator.webdriver !== false
})
} else {
// Detects the --enable-automation || --headless flags
// Will return true in headful if --enable-automation is provided
await test('navigator.webdriver present', _ => {
return 'webdriver' in navigator
})
await test('navigator.webdriver not undefined', _ => {
return navigator.webdriver !== undefined
})
/* eslint-disable no-proto */
await test('navigator.webdriver property overridden', _ => {
return (
Object.getOwnPropertyDescriptor(navigator.__proto__, 'webdriver') !==
undefined
)
})
await test('navigator.webdriver prop detected', _ => {
for (const prop in navigator) {
if (prop === 'webdriver') {
return true
}
}
return false
})
}
await test('window.chrome missing', _ => {
return /Chrome/.test(window.navigator.userAgent) && !window.chrome
})
await test('permissions API', async _ => {
const permissionStatus = await navigator.permissions.query({
name: 'notifications'
})
return (
Notification.permission === 'denied' &&
permissionStatus.state === 'prompt'
)
})
await test('permissions API overriden', _ => {
const permissions = window.navigator.permissions
if (permissions.query.toString() !== 'function query() { [native code] }')
return true
if (
permissions.query.toString.toString() !==
'function toString() { [native code] }'
)
return true
if (
permissions.query.toString.hasOwnProperty('[[Handler]]') && // eslint-disable-line
permissions.query.toString.hasOwnProperty('[[Target]]') && // eslint-disable-line
permissions.query.toString.hasOwnProperty('[[IsRevoked]]') // eslint-disable-line
)
return true
if (permissions.hasOwnProperty('query')) return true // eslint-disable-line
})
await test('navigator.plugins empty', _ => {
return navigator.plugins.length === 0
})
await test('navigator.languages blank', _ => {
return navigator.languages === ''
})
await test('iFrame for fresh window object', _ => {
// evaluateOnNewDocument scripts don't apply within [srcdoc] (or [sandbox]) iframes
// https://github.com/GoogleChrome/puppeteer/issues/1106#issuecomment-359313898
const iframe = document.createElement('iframe')
iframe.srcdoc = 'page intentionally left blank'
document.body.appendChild(iframe)
// Verify iframe prototype isn't touched
const descriptors = Object.getOwnPropertyDescriptors(
HTMLIFrameElement.prototype
)
if (
descriptors.contentWindow.get.toString() !==
'function get contentWindow() { [native code] }'
)
return true
// Verify iframe isn't remapped to main window
if (iframe.contentWindow === window) return true
// Here we would need to rerun all tests with `iframe.contentWindow` as `window`
// Example:
return iframe.contentWindow.navigator.plugins.length === 0
})
// This detects that a devtools protocol agent is attached.
// So it will also pass true in headful Chrome if the devtools window is attached
await test('toString', _ => {
let gotYou = 0
const spooky = /./
spooky.toString = function() {
gotYou++
return 'spooky'
}
console.debug(spooky)
return gotYou > 1
})
return results
}

View File

@@ -0,0 +1,22 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<title>title foo</title>
<!-- Testing evasions with a real html page makes things easier -->
<script>
if ('serviceWorker' in navigator) {
window.addEventListener('load', function() {
navigator.serviceWorker.register('/sw.js').then(function(registration) {
console.log('ServiceWorker registration successful with scope: ', registration.scope);
}, function(err) {
console.log('ServiceWorker registration failed: ', err);
});
});
}
</script>
</head>
<body>
<h1>Test page with service worker</h1>
</body>
</html>

View File

@@ -0,0 +1,11 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<title>title foo</title>
<!-- Testing evasions with a real html page makes things easier -->
</head>
<body>
<h1>Test page</h1>
</body>
</html>

View File

@@ -0,0 +1 @@
// Left empty

View File

@@ -0,0 +1,52 @@
const test = require('ava')
const fpscanner = require('fpscanner')
const { getVanillaFingerPrint, getStealthFingerPrint, compareLooseVersionStrings } = require('./util')
const Plugin = require('../.')
// Fix CI issues with old versions
const isOldPuppeteerVersion = () => {
const version = process.env.PUPPETEER_VERSION
if (!version) {
return false
}
if (version === '1.9.0' || version === '1.6.2') {
return true
}
return false
}
test('vanilla: will fail multiple fpscanner tests', async t => {
const fingerPrint = await getVanillaFingerPrint()
const testedFingerPrints = fpscanner.analyseFingerprint(fingerPrint)
const failedChecks = Object.values(testedFingerPrints).filter(
val => val.consistent < 3
)
if (isOldPuppeteerVersion()) {
t.is(failedChecks.length, 8)
} else {
t.is(failedChecks.length, 7)
}
})
test('stealth: will not fail a single fpscanner test', async t => {
const fingerPrint = await getStealthFingerPrint(Plugin)
const testedFingerPrints = fpscanner.analyseFingerprint(fingerPrint)
const failedChecks = Object.values(testedFingerPrints).filter(
val => val.consistent < 3
)
if (failedChecks.length) {
console.warn('The following fingerprints failed:', failedChecks)
}
if (compareLooseVersionStrings(fingerPrint.userAgent, '89.0.4339.0') >= 0) {
// Updated navigator.webdriver behavior breaks the fpscanner tests.
t.is(failedChecks.length, 1)
t.is(failedChecks[0].name, 'WEBDRIVER')
} else {
t.is(failedChecks.length, 0)
}
})

View File

@@ -0,0 +1,112 @@
const test = require('ava')
const { vanillaPuppeteer, addExtra } = require('./util')
const Plugin = require('..')
const http = require('http')
const fs = require('fs')
const path = require('path')
// Create a simple HTTP server. Service Workers cannot be served from file:// URIs
const httpServer = async () => {
const server = await http
.createServer((req, res) => {
let contents, type
if (req.url === '/sw.js') {
contents = fs.readFileSync(path.join(__dirname, './fixtures/sw.js'))
type = 'application/javascript'
} else {
contents = fs.readFileSync(
path.join(__dirname, './fixtures/dummy-with-service-worker.html')
)
type = 'text/html'
}
res.setHeader('Content-Type', type)
res.writeHead(200)
res.end(contents)
})
.listen(0) // random free port
return `http://127.0.0.1:${server.address().port}/`
}
let browser, page, worker
test.before(async t => {
const address = await httpServer()
console.log(`Server is running on port ${address}`)
browser = await addExtra(vanillaPuppeteer)
.use(Plugin())
.launch({ headless: true })
page = await browser.newPage()
worker = new Promise(resolve => {
browser.on('targetcreated', async target => {
if (target.type() === 'service_worker') {
resolve(target.worker())
}
})
})
await page.goto(address)
worker = await worker
})
test.after(async t => {
await browser.close()
})
test.skip('stealth: inconsistencies between page and worker', async t => {
const pageFP = await page.evaluate(detectFingerprint)
const workerFP = await worker.evaluate(detectFingerprint)
t.deepEqual(pageFP, workerFP)
})
test.serial.skip('stealth: creepjs has good trust score', async t => {
page.goto('https://abrahamjuliot.github.io/creepjs/')
const score = await (
await (
await page.waitForSelector('#fingerprint-data .unblurred')
).getProperty('textContent')
).jsonValue()
t.true(
parseInt(score) > 80,
`The creepjs score is: ${parseInt(score)}% but it should be at least 80%`
)
})
/* global OffscreenCanvas */
function detectFingerprint() {
const results = {}
const props = [
'userAgent',
'language',
'hardwareConcurrency',
'deviceMemory',
'languages',
'platform'
]
props.forEach(el => {
results[el] = navigator[el].toString()
})
const canvasOffscreenWebgl = new OffscreenCanvas(256, 256)
const contextWebgl = canvasOffscreenWebgl.getContext('webgl')
const rendererInfo = contextWebgl.getExtension('WEBGL_debug_renderer_info')
results.webglVendor = contextWebgl.getParameter(
rendererInfo.UNMASKED_VENDOR_WEBGL
)
results.webglRenderer = contextWebgl.getParameter(
rendererInfo.UNMASKED_RENDERER_WEBGL
)
results.timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone
return results
}

View File

@@ -0,0 +1,65 @@
const assert = require('assert')
const vanillaPuppeteer = require('puppeteer')
const { addExtra } = require('puppeteer-extra')
const fpCollectPath = require.resolve('fpcollect/dist/fpCollect.min.js')
const getFingerPrintFromPage = async page => {
return page.evaluate(() => fpCollect.generateFingerprint()) // eslint-disable-line
}
const dummyHTMLPath = require('path').join(__dirname, './fixtures/dummy.html')
const getFingerPrint = async (puppeteer, pageFn) => {
const browser = await puppeteer.launch({ headless: true })
const page = await browser.newPage()
await page.goto('file://' + dummyHTMLPath)
await page.addScriptTag({ path: fpCollectPath })
const fingerPrint = await getFingerPrintFromPage(page)
let pageFnResult = null
if (pageFn) {
pageFnResult = await pageFn(page)
}
await browser.close()
return { ...fingerPrint, pageFnResult }
}
const getVanillaFingerPrint = async pageFn =>
getFingerPrint(vanillaPuppeteer, pageFn)
const getStealthFingerPrint = async (Plugin, pageFn, pluginOptions = null) =>
getFingerPrint(addExtra(vanillaPuppeteer).use(Plugin(pluginOptions)), pageFn)
// Expecting the input string to be in one of these formats:
// - The UA string
// - The shorter version string from Puppeteers browser.version()
// - The shortest four-integer string
const parseLooseVersionString = looseVersionString => looseVersionString
.match(/(\d+\.){3}\d+/)[0]
.split('.')
.map(x => parseInt(x))
const compareLooseVersionStrings = (version0, version1) => {
const parsed0 = parseLooseVersionString(version0)
const parsed1 = parseLooseVersionString(version1)
assert(parsed0.length == 4)
assert(parsed1.length == 4)
for (let i = 0; i < parsed0.length; i++) {
if (parsed0[i] < parsed1[i]) {
return -1
} else if (parsed0[i] > parsed1[i]) {
return 1
}
}
return 0
}
module.exports = {
getVanillaFingerPrint,
getStealthFingerPrint,
dummyHTMLPath,
vanillaPuppeteer,
addExtra,
compareLooseVersionStrings
}