Rename to hkt.sh

2026-03-21 01:10:53 +08:00
parent 76a263d0f9
commit 8f1171fe99
6676 changed files with 1724268 additions and 0 deletions
--- a/node_modules/puppeteer-extra-plugin-stealth/evasions/user-agent-override/index.js
+++ b/node_modules/puppeteer-extra-plugin-stealth/evasions/user-agent-override/index.js
@@ -0,0 +1,208 @@
+'use strict'
+
+const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
+
+/**
+ * Fixes the UserAgent info (composed of UA string, Accept-Language, Platform, and UA hints).
+ *
+ * If you don't provide any values this plugin will default to using the regular UserAgent string (while stripping the headless part).
+ * Default language is set to "en-US,en", the other settings match the UserAgent string.
+ * If you are running on Linux, it will mask the settins to look like Windows. This behavior can be disabled with the `maskLinux` option.
+ *
+ * By default puppeteer will not set a `Accept-Language` header in headless:
+ * It's (theoretically) possible to fix that using either `page.setExtraHTTPHeaders` or a `--lang` launch arg.
+ * Unfortunately `page.setExtraHTTPHeaders` will lowercase everything and launch args are not always available. :)
+ *
+ * In addition, the `navigator.platform` property is always set to the host value, e.g. `Linux` which makes detection very easy.
+ *
+ * Note: You cannot use the regular `page.setUserAgent()` puppeteer call in your code,
+ * as it will reset the language and platform values you set with this plugin.
+ *
+ * @example
+ * const puppeteer = require("puppeteer-extra")
+ *
+ * const StealthPlugin = require("puppeteer-extra-plugin-stealth")
+ * const stealth = StealthPlugin()
+ * // Remove this specific stealth plugin from the default set
+ * stealth.enabledEvasions.delete("user-agent-override")
+ * puppeteer.use(stealth)
+ *
+ * // Stealth plugins are just regular `puppeteer-extra` plugins and can be added as such
+ * const UserAgentOverride = require("puppeteer-extra-plugin-stealth/evasions/user-agent-override")
+ * // Define custom UA and locale
+ * const ua = UserAgentOverride({ userAgent: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)", locale: "de-DE,de" })
+ * puppeteer.use(ua)
+ *
+ * @param {Object} [opts] - Options
+ * @param {string} [opts.userAgent] - The user agent to use (default: browser.userAgent())
+ * @param {string} [opts.locale] - The locale to use in `Accept-Language` header and in `navigator.languages` (default: `en-US,en`)
+ * @param {boolean} [opts.maskLinux] - Wether to hide Linux as platform in the user agent or not - true by default
+ *
+ */
+class Plugin extends PuppeteerExtraPlugin {
+  constructor(opts = {}) {
+    super(opts)
+
+    this._headless = false
+  }
+
+  get name() {
+    return 'stealth/evasions/user-agent-override'
+  }
+
+  get dependencies() {
+    return new Set(['user-preferences'])
+  }
+
+  get defaults() {
+    return {
+      userAgent: null,
+      locale: 'en-US,en',
+      maskLinux: true
+    }
+  }
+
+  async onPageCreated(page) {
+    // Determine the full user agent string, strip the "Headless" part
+    let ua =
+      this.opts.userAgent ||
+      (await page.browser().userAgent()).replace('HeadlessChrome/', 'Chrome/')
+
+    if (
+      this.opts.maskLinux &&
+      ua.includes('Linux') &&
+      !ua.includes('Android') // Skip Android user agents since they also contain Linux
+    ) {
+      ua = ua.replace(/\(([^)]+)\)/, '(Windows NT 10.0; Win64; x64)') // Replace the first part in parentheses with Windows data
+    }
+
+    // Full version number from Chrome
+    const uaVersion = ua.includes('Chrome/')
+      ? ua.match(/Chrome\/([\d|.]+)/)[1]
+      : (await page.browser().version()).match(/\/([\d|.]+)/)[1]
+
+    // Get platform identifier (short or long version)
+    const _getPlatform = (extended = false) => {
+      if (ua.includes('Mac OS X')) {
+        return extended ? 'Mac OS X' : 'MacIntel'
+      } else if (ua.includes('Android')) {
+        return 'Android'
+      } else if (ua.includes('Linux')) {
+        return 'Linux'
+      } else {
+        return extended ? 'Windows' : 'Win32'
+      }
+    }
+
+    // Source in C++: https://source.chromium.org/chromium/chromium/src/+/master:components/embedder_support/user_agent_utils.cc;l=55-100
+    const _getBrands = () => {
+      const seed = uaVersion.split('.')[0] // the major version number of Chrome
+
+      const order = [
+        [0, 1, 2],
+        [0, 2, 1],
+        [1, 0, 2],
+        [1, 2, 0],
+        [2, 0, 1],
+        [2, 1, 0]
+      ][seed % 6]
+      const escapedChars = [' ', ' ', ';']
+
+      const greaseyBrand = `${escapedChars[order[0]]}Not${
+        escapedChars[order[1]]
+      }A${escapedChars[order[2]]}Brand`
+
+      const greasedBrandVersionList = []
+      greasedBrandVersionList[order[0]] = {
+        brand: greaseyBrand,
+        version: '99'
+      }
+      greasedBrandVersionList[order[1]] = {
+        brand: 'Chromium',
+        version: seed
+      }
+      greasedBrandVersionList[order[2]] = {
+        brand: 'Google Chrome',
+        version: seed
+      }
+
+      return greasedBrandVersionList
+    }
+
+    // Return OS version
+    const _getPlatformVersion = () => {
+      if (ua.includes('Mac OS X ')) {
+        return ua.match(/Mac OS X ([^)]+)/)[1]
+      } else if (ua.includes('Android ')) {
+        return ua.match(/Android ([^;]+)/)[1]
+      } else if (ua.includes('Windows ')) {
+        return ua.match(/Windows .*?([\d|.]+);?/)[1]
+      } else {
+        return ''
+      }
+    }
+
+    // Get architecture, this seems to be empty on mobile and x86 on desktop
+    const _getPlatformArch = () => (_getMobile() ? '' : 'x86')
+
+    // Return the Android model, empty on desktop
+    const _getPlatformModel = () =>
+      _getMobile() ? ua.match(/Android.*?;\s([^)]+)/)[1] : ''
+
+    const _getMobile = () => ua.includes('Android')
+
+    const override = {
+      userAgent: ua,
+      platform: _getPlatform(),
+      userAgentMetadata: {
+        brands: _getBrands(),
+        fullVersion: uaVersion,
+        platform: _getPlatform(true),
+        platformVersion: _getPlatformVersion(),
+        architecture: _getPlatformArch(),
+        model: _getPlatformModel(),
+        mobile: _getMobile()
+      }
+    }
+
+    // In case of headless, override the acceptLanguage in CDP.
+    // This is not preferred, as it messed up the header order.
+    // On headful, we set the user preference language setting instead.
+    if (this._headless) {
+      override.acceptLanguage = this.opts.locale || 'en-US,en'
+    }
+
+    this.debug('onPageCreated - Will set these user agent options', {
+      override,
+      opts: this.opts
+    })
+
+    const client =
+      typeof page._client === 'function' ? page._client() : page._client
+    client.send('Network.setUserAgentOverride', override)
+  }
+
+  async beforeLaunch(options) {
+    // Check if launched headless
+    this._headless = options.headless
+  }
+
+  async beforeConnect() {
+    // Treat browsers using connect() as headless browsers
+    this._headless = true
+  }
+
+  get data() {
+    return [
+      {
+        name: 'userPreferences',
+        value: {
+          intl: { accept_languages: this.opts.locale || 'en-US,en' }
+        }
+      }
+    ]
+  }
+}
+
+const defaultExport = opts => new Plugin(opts)
+module.exports = defaultExport
--- a/node_modules/puppeteer-extra-plugin-stealth/evasions/user-agent-override/index.test.js
+++ b/node_modules/puppeteer-extra-plugin-stealth/evasions/user-agent-override/index.test.js
@@ -0,0 +1,324 @@
+const test = require('ava')
+
+const { vanillaPuppeteer, addExtra } = require('../../test/util')
+const Plugin = require('.')
+
+// Fixed since 2.1.1?
+// test('vanilla: Accept-Language header is missing', async t => {
+//   const browser = await vanillaPuppeteer.launch({ headless: true })
+//   const page = await browser.newPage()
+//   await page.goto('http://httpbin.org/headers')
+
+//   const content = await page.content()
+//   t.true(content.includes(`"User-Agent"`))
+//   t.false(content.includes(`"Accept-Language"`))
+// })
+
+test('vanilla: User-Agent header contains HeadlessChrome', async t => {
+  const browser = await vanillaPuppeteer.launch({ headless: true })
+  const page = await browser.newPage()
+  await page.goto('http://httpbin.org/headers')
+
+  const content = await page.content()
+  t.true(content.includes(`"User-Agent"`))
+  t.true(content.includes(`HeadlessChrome`))
+})
+
+test('vanilla: navigator.languages is always en-US', async t => {
+  const browser = await vanillaPuppeteer.launch({ headless: true })
+  const page = await browser.newPage()
+  const lang = await page.evaluate(() => navigator.languages)
+  t.true(lang.length === 1 && lang[0] === 'en-US')
+})
+
+test('vanilla: navigator.platform set to host platform', async t => {
+  const browser = await vanillaPuppeteer.launch({ headless: true })
+  const page = await browser.newPage()
+
+  const platform = await page.evaluate(() => navigator.platform)
+  switch (process.platform) {
+    case 'linux':
+      t.true(platform.includes('Linux')) // TravisCI
+      break
+    case 'darwin':
+      t.true(platform === 'MacIntel')
+      break
+    case 'win32':
+      t.true(platform === 'Win32')
+      break
+    default:
+      t.true(platform === process.platform)
+  }
+})
+
+test('stealth: Accept-Language header with default locale', async t => {
+  const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
+  const browser = await puppeteer.launch({ headless: true })
+  const page = await browser.newPage()
+  await page.goto('http://httpbin.org/headers')
+
+  const content = await page.content()
+  t.true(content.includes(`"User-Agent"`))
+  t.true(content.includes(`"Accept-Language": "en-US,en;q=0.9"`))
+})
+
+test('stealth: Accept-Language header with optional locale', async t => {
+  const puppeteer = addExtra(vanillaPuppeteer).use(
+    Plugin({ locale: 'de-DE,de' })
+  )
+  const browser = await puppeteer.launch({ headless: true })
+  const page = await browser.newPage()
+  await page.goto('http://httpbin.org/headers')
+
+  const content = await page.content()
+  t.true(content.includes(`"User-Agent"`))
+  t.true(content.includes(`"Accept-Language": "de-DE,de;q=0.9"`))
+})
+
+test('stealth: User-Agent header does not contain HeadlessChrome', async t => {
+  const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
+  const browser = await puppeteer.launch({ headless: true })
+  const page = await browser.newPage()
+  await page.goto('http://httpbin.org/headers')
+
+  const content = await page.content()
+  t.true(content.includes(`"User-Agent"`))
+  t.false(content.includes(`HeadlessChrome`))
+})
+
+test('stealth: User-Agent header with custom userAgent', async t => {
+  const puppeteer = addExtra(vanillaPuppeteer).use(
+    Plugin({ userAgent: 'MyFunkyUA/1.0' })
+  )
+  const browser = await puppeteer.launch({ headless: true })
+  const page = await browser.newPage()
+  await page.goto('http://httpbin.org/headers')
+
+  const content = await page.content()
+  t.true(content.includes(`"User-Agent": "MyFunkyUA/1.0"`))
+})
+
+test('stealth: navigator.languages with default locale', async t => {
+  const puppeteer = addExtra(vanillaPuppeteer).use(Plugin())
+  const browser = await puppeteer.launch({ headless: true })
+  const page = await browser.newPage()
+
+  const lang = await page.evaluate(() => navigator.languages)
+  t.true(lang.length === 2 && lang[0] === 'en-US' && lang[1] === 'en')
+})
+
+test('stealth: navigator.languages with custom locale', async t => {
+  const puppeteer = addExtra(vanillaPuppeteer).use(
+    Plugin({ locale: 'de-DE,de' })
+  )
+  const browser = await puppeteer.launch({ headless: true })
+  const page = await browser.newPage()
+
+  const langs = await page.evaluate(() => navigator.languages)
+  t.deepEqual(langs, ['de-DE', 'de'])
+  const lang = await page.evaluate(() => navigator.language)
+  t.deepEqual(lang, 'de-DE')
+})
+
+test('stealth: navigator.platform with maskLinux true (default)', async t => {
+  const puppeteer = addExtra(vanillaPuppeteer).use(
+    Plugin({
+      userAgent:
+        'Mozilla/5.0 (X11; Ubuntu; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.9.9999.99 Safari/537.36'
+    })
+  )
+  const browser = await puppeteer.launch({ headless: true })
+  const page = await browser.newPage()
+
+  const platform = await page.evaluate(() => navigator.platform)
+  t.true(platform === 'Win32')
+})
+
+test('stealth: navigator.platform with maskLinux false', async t => {
+  const puppeteer = addExtra(vanillaPuppeteer).use(
+    Plugin({
+      userAgent:
+        'Mozilla/5.0 (X11; Ubuntu; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.9.9999.99 Safari/537.36',
+      maskLinux: false
+    })
+  )
+  const browser = await puppeteer.launch({ headless: true })
+  const page = await browser.newPage()
+
+  const platform = await page.evaluate(() => navigator.platform)
+  t.true(platform === 'Linux')
+})
+
+const _testUAHint = async (userAgent, locale) => {
+  const puppeteer = addExtra(vanillaPuppeteer).use(
+    Plugin({ userAgent, locale })
+  )
+
+  const browser = await puppeteer.launch({
+    headless: false, // only works on headful
+    args: ['--enable-features=UserAgentClientHint']
+  })
+
+  const majorVersion = parseInt(
+    (await browser.version()).match(/\/([^\.]+)/)[1]
+  )
+  if (majorVersion < 88) {
+    return null // Skip test on browsers that don't support UA hints
+  }
+
+  const page = await browser.newPage()
+
+  await page.goto('https://headers.cf/headers/?format=raw')
+
+  return page
+}
+
+test('stealth: test if UA hints are correctly set - Windows 10', async t => {
+  const page = await _testUAHint(
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.9999.99 Safari/537.36',
+    'en-AU'
+  )
+  if (!page) {
+    t.true(true) // skip
+    return
+  }
+  const firstLoad = await page.content()
+  t.true(
+    firstLoad.includes(
+      `sec-ch-ua: "Google Chrome";v="99", " Not;A Brand";v="99", "Chromium";v="99"`
+    )
+  )
+  t.true(firstLoad.includes(`Accept-Language: en-AU`))
+
+  await page.reload()
+  const secondLoad = await page.content()
+  if (secondLoad.includes('sec-ch-ua-full-version')) {
+    t.true(secondLoad.includes('sec-ch-ua-mobile: ?0'))
+    t.true(secondLoad.includes('sec-ch-ua-full-version: "99.0.9999.99"'))
+    t.true(secondLoad.includes('sec-ch-ua-arch: "x86"'))
+    t.true(secondLoad.includes('sec-ch-ua-platform: "Windows"'))
+    t.true(secondLoad.includes('sec-ch-ua-platform-version: "10.0"'))
+    t.true(secondLoad.includes('sec-ch-ua-model: ""'))
+  }
+})
+
+test('stealth: test if UA hints are correctly set - macOS 11', async t => {
+  const page = await _testUAHint(
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.9999.99 Safari/537.36',
+    'de-DE'
+  )
+  if (!page) {
+    t.true(true) // skip
+    return
+  }
+  const firstLoad = await page.content()
+  t.true(
+    firstLoad.includes(
+      `sec-ch-ua: "Google Chrome";v="99", " Not;A Brand";v="99", "Chromium";v="99"`
+    )
+  )
+  t.true(firstLoad.includes(`Accept-Language: de-DE`))
+
+  await page.reload()
+  const secondLoad = await page.content()
+  if (secondLoad.includes('sec-ch-ua-full-version')) {
+    t.true(secondLoad.includes('sec-ch-ua-mobile: ?0'))
+    t.true(secondLoad.includes('sec-ch-ua-full-version: "99.0.9999.99"'))
+    t.true(secondLoad.includes('sec-ch-ua-arch: "x86"'))
+    t.true(secondLoad.includes('sec-ch-ua-platform: "Mac OS X"'))
+    t.true(secondLoad.includes('sec-ch-ua-platform-version: "11_1_0"'))
+    t.true(secondLoad.includes('sec-ch-ua-model: ""'))
+  }
+})
+
+test('stealth: test if UA hints are correctly set - Android 10', async t => {
+  const page = await _testUAHint(
+    'Mozilla/5.0 (Linux; Android 10; SM-P205) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.9999.99 Safari/537.36',
+    'nl-NL'
+  )
+  if (!page) {
+    t.true(true) // skip
+    return
+  }
+  const firstLoad = await page.content()
+  t.true(
+    firstLoad.includes(
+      `sec-ch-ua: "Google Chrome";v="99", " Not;A Brand";v="99", "Chromium";v="99"`
+    )
+  )
+  t.true(firstLoad.includes(`Accept-Language: nl-NL`))
+
+  await page.reload()
+  const secondLoad = await page.content()
+
+  if (secondLoad.includes('sec-ch-ua-full-version')) {
+    t.true(secondLoad.includes('sec-ch-ua-mobile: ?1'))
+    t.true(secondLoad.includes('sec-ch-ua-full-version: "99.0.9999.99"'))
+    t.true(secondLoad.includes('sec-ch-ua-arch: ""'))
+    t.true(secondLoad.includes('sec-ch-ua-platform: "Android"'))
+    t.true(secondLoad.includes('sec-ch-ua-platform-version: "10"'))
+    t.true(secondLoad.includes('sec-ch-ua-model: "SM-P205"'))
+  }
+})
+
+async function userAgentData() {
+  if (!('userAgentData' in navigator)) {
+    return undefined
+  }
+
+  // https://wicg.github.io/ua-client-hints/#getHighEntropyValues
+  const UADataProps = ['brands', 'mobile']
+  const UADataValues = [
+    'architecture', // "arm"
+    'bitness', // "64"
+    'model', // "X644GTM"
+    'platform', // "PhoneOS"
+    'platformVersion', // "10A"
+    'uaFullVersion' // "73.32.AGX.5"
+  ]
+
+  const highEntropy = await navigator.userAgentData.getHighEntropyValues(
+    UADataValues
+  )
+
+  const result = {
+    ...highEntropy,
+    ...Object.fromEntries(UADataProps.map(k => [k, navigator.userAgentData[k]]))
+  }
+  return result
+}
+
+test('stealth: test if UA hints are correctly set - Windows 10 Generic', async t => {
+  const userAgent =
+    'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.9999.99 Safari/537.36'
+  const locale = 'en-AU'
+
+  const puppeteer = addExtra(vanillaPuppeteer).use(
+    Plugin({
+      userAgent,
+      locale
+    })
+  )
+  const browser = await puppeteer.launch({
+    headless: true
+  })
+
+  const majorVersion = parseInt(
+    (await browser.version()).match(/\/([^\.]+)/)[1]
+  )
+  if (majorVersion < 90) {
+    t.truthy('foo')
+    console.log('Skipping test, browser version too old', majorVersion)
+    return
+  }
+  const page = await browser.newPage()
+  await page.goto('https://example.com') // secure context
+
+  const results = await page.evaluate(userAgentData)
+  t.is(results.platform, 'Windows')
+  t.is(results.platformVersion, '10.0')
+  t.is(results.uaFullVersion, '99.0.9999.99')
+
+  const language = await page.evaluate(() => navigator.language)
+  t.is(language, locale)
+})
--- a/node_modules/puppeteer-extra-plugin-stealth/evasions/user-agent-override/package.json
+++ b/node_modules/puppeteer-extra-plugin-stealth/evasions/user-agent-override/package.json
@@ -0,0 +1,4 @@
+{
+  "private": true,
+  "main": "index.js"
+}
--- a/node_modules/puppeteer-extra-plugin-stealth/evasions/user-agent-override/readme.md
+++ b/node_modules/puppeteer-extra-plugin-stealth/evasions/user-agent-override/readme.md
@@ -0,0 +1,54 @@
+## API
+
+<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
+
+#### Table of Contents
+
+- [class: Plugin](#class-plugin)
+
+### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/ab0047d1af7dc38412744abdb61bcfc35c42dc34/packages/puppeteer-extra-plugin-stealth/evasions/user-agent-override/index.js#L42-L203)
+
+- `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`)
+  - `opts.userAgent` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The user agent to use (default: browser.userAgent())
+  - `opts.locale` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The locale to use in `Accept-Language` header and in `navigator.languages` (default: `en-US,en`)
+  - `opts.maskLinux` **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)?** Wether to hide Linux as platform in the user agent or not - true by default
+
+**Extends: PuppeteerExtraPlugin**
+
+Fixes the UserAgent info (composed of UA string, Accept-Language, Platform, and UA hints).
+
+If you don't provide any values this plugin will default to using the regular UserAgent string (while stripping the headless part).
+Default language is set to "en-US,en", the other settings match the UserAgent string.
+If you are running on Linux, it will mask the settins to look like Windows. This behavior can be disabled with the `maskLinux` option.
+
+By default puppeteer will not set a `Accept-Language` header in headless:
+It's (theoretically) possible to fix that using either `page.setExtraHTTPHeaders` or a `--lang` launch arg.
+Unfortunately `page.setExtraHTTPHeaders` will lowercase everything and launch args are not always available. :)
+
+In addition, the `navigator.platform` property is always set to the host value, e.g. `Linux` which makes detection very easy.
+
+Note: You cannot use the regular `page.setUserAgent()` puppeteer call in your code,
+as it will reset the language and platform values you set with this plugin.
+
+Example:
+
+```javascript
+const puppeteer = require('puppeteer-extra')
+
+const StealthPlugin = require('puppeteer-extra-plugin-stealth')
+const stealth = StealthPlugin()
+// Remove this specific stealth plugin from the default set
+stealth.enabledEvasions.delete('user-agent-override')
+puppeteer.use(stealth)
+
+// Stealth plugins are just regular `puppeteer-extra` plugins and can be added as such
+const UserAgentOverride = require('puppeteer-extra-plugin-stealth/evasions/user-agent-override')
+// Define custom UA and locale
+const ua = UserAgentOverride({
+  userAgent: 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)',
+  locale: 'de-DE,de'
+})
+puppeteer.use(ua)
+```
+
+---