diff --git a/.gitignore b/.gitignore index 8aef2b7..de4d1f0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ dist -result node_modules diff --git a/README.md b/README.md index 20c3acf..ad8ec63 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # crux-api -> A [Chrome UX Report API](https://developers.google.com/web/tools/chrome-user-experience-report/api/reference) wrapper that supports batching, handles errors, and provides types. +> A [Chrome UX Report API](https://developers.google.com/web/tools/chrome-user-experience-report/api/reference) wrapper that handles errors, retries, and provides types. **Motivation**: [CrUX API](https://web.dev/chrome-ux-report-api/) is a fantastic tool to get RUM data without installing any script. While using the API in [Treo](https://treo.sh/), we discovered a few complex cases like API errors, rate limits, not found entries, a complicated multipart response from the batch API, URLs normalization, and TypeScript notations. We decided to build the `crux-api` library to makes it easier to work with the CrUX API. @@ -8,7 +8,6 @@ While using the API in [Treo](https://treo.sh/), we discovered a few complex cas **Features**: - A tiny (450 bytes) wrapper for [Chrome UX Report API](https://developers.google.com/web/tools/chrome-user-experience-report/api/reference); -- [Batch API support](https://developers.google.com/web/tools/chrome-user-experience-report/api/guides/batch) for up to 1000 records per one request; - TypeScript notations for [options and responses](https://developers.google.com/web/tools/chrome-user-experience-report/api/reference/rest/v1/records/queryRecord); - Isomorphic: works in a browser and node.js; - Returns `null` for the `404 (CrUX data not found)` response; @@ -33,20 +32,6 @@ const res1 = await queryRecord({ url: 'https://www.github.com/' }) // fetch all const res2 = await queryRecord({ url: 'https://www.github.com/explore', formFactor: 'DESKTOP' }) // fetch data for desktop devices ``` -Use the [CrUX Batch API](https://developers.google.com/web/tools/chrome-user-experience-report/api/guides/batch) to combine up to 1000 requests and get results in less than 1 second: - -```js -import { createBatch } from 'crux-api/batch' -const batch = createBatch({ key: CRUX_API_KEY }) - -const records = await batch([ - { url: 'https://github.com/', formFactor: 'MOBILE', effectiveConnectionType: '4G' }, - { url: 'https://github.com/marketplace', formFactor: 'DESKTOP' }, - { url: 'https://www.github.com/explore', formFactor: 'TABLET' }, - // ... up to 1000 records. -]) -``` - Fetch origin-level data in node.js using [`node-fetch`](https://www.npmjs.com/package/node-fetch): ```js @@ -97,9 +82,7 @@ Result is `null` or an `object` with [queryRecord response body](https://develop ## API -### Single Record Request - -#### createQueryRecord(createOptions) +### createQueryRecord(createOptions) Returns a `queryRecord` function. @@ -129,36 +112,6 @@ const res = await queryRecord({ // res -> URL-level data for https://github.com/marketplace ``` -### Batch Request - -`crux-api/batch` uses the [CrUX Batch API](https://developers.google.com/web/tools/chrome-user-experience-report/api/guides/batch), which allows combining 1000 calls in a single batch request. It's a separate namespace because the API is different, and it's bigger (850 bytes) due to the complexity of constructing and parsing multipart requests. - -_Note_: A set of `n` requests batched together counts toward your usage limit as `n` requests, not as one request. That's why the sometimes a batch response contains `429` responses. But the `crux-api` automatically retries these responses, aiming always to return the data you need. - -#### createBatch(createOptions) - -Accepts the same [`createOptions` as the `createQueryRecord`](#createqueryrecordcreateoptions) and returns a `batch` function. - -#### batch(batchOptions) - -Accepts an array of [`queryRecord`](#queryrecordqueryoptions) options and returns an array with an exact position for each record. If the record doesn't exist in CrUX index, the value set to null. If some requests hit rate-limit, `batch` will retry them after a short timeout. - -```js -import { createBatch } from 'crux-api/batch' -import nodeFetch from 'node-fetch' - -const batch = createBatch({ key: process.env.CRUX_KEY, fetch: nodeFetch }) -const res = await batch([ - { origin: 'https://example.com' }, - { url: 'https://github.com/', formFactor: 'DESKTOP' }, - { origin: 'https://fooo.bar' }, -]) - -// res[0] -> origin-level data for https://example.com -// res[1] -> URL-level data for https://github.com/ on desktop devices -// res[2] -> null (invalid origin that not found in the CrUX index) -``` - ### normalizeUrl(url) Normalize a URL to match the CrUX API internal index. diff --git a/batch/package.json b/batch/package.json deleted file mode 100644 index 5cfae1d..0000000 --- a/batch/package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "crux-api-batch", - "version": "0.0.0", - "description": "Batch support for Chrome UX Report API", - "private": true, - "license": "MIT", - "source": "src/index.js", - "module": "src/index.js", - "types": "dist/crux-api-batch.d.ts", - "main": "dist/crux-api-batch.js" -} diff --git a/batch/src/index.js b/batch/src/index.js deleted file mode 100644 index f81d1c3..0000000 --- a/batch/src/index.js +++ /dev/null @@ -1,261 +0,0 @@ -import { retryAfterTimeout } from '../../src/retry' -const boundary = 'BATCH_BOUNDARY' - -/** - * Create batch interface for CrUX API. - * https://developers.google.com/web/tools/chrome-user-experience-report/api/guides/batch - * - * @typedef {{ options: import('../..').QueryRecordOptions, result: import('../..').SuccessResponse | null | undefined }[]} BatchValues - * - * @param {import('../..').CreateOptions} createOptions - */ - -export function createBatch(createOptions) { - const key = createOptions.key - const fetch = createOptions.fetch || window.fetch - return batch - - /** - * @param {import('../..').BatchOptions} batchOptions - * - */ - - function batch(batchOptions) { - const batchValues = /** @type {BatchValues} */ (batchOptions.map((options) => ({ options, result: undefined }))) - return batchRequest(1) - - /** - * @param {number} retryCounter - * @return {Promise} - */ - - async function batchRequest(retryCounter) { - const body = generateBatchBody(batchValues, key) - const res = await fetch('https://chromeuxreport.googleapis.com/batch/', { - method: 'POST', - headers: { 'Content-Type': `multipart/mixed; boundary=${boundary}` }, - body, - }) - const text = await res.text() - if (res.status !== 200) throw new Error(`Invalid batch response: ${text}`) - const results = parseBatchResponse(text) - results.forEach(({ index, json }) => { - if (!json) { - throw new Error('Empty result') - } else if (json.error) { - const { error } = /** @type {import('../..').ErrorResponse} */ (json) - if (error.code === 404) { - batchValues[index].result = null - } else if (error.code !== 429) { - throw new Error(JSON.stringify(error)) - } - } else { - batchValues[index].result = json - } - }) - const rateLimitedRequests = batchValues.filter(({ result }) => result === undefined) - if (rateLimitedRequests.length) { - console.log('Rate-limit #%s: %s/%s', retryCounter, rateLimitedRequests.length, results.length) - return retryAfterTimeout(retryCounter, () => batchRequest(retryCounter + 1)) - } - return batchValues.map(({ result }) => /** @type {import('../..').SuccessResponse | null} */ (result)) - } - } -} - -/** - * Generate multipart body for batch request. - * - * Example for `[{ origin: "https://github.com" }]`: - * - * --BATCH_BOUNDARY - * Content-Type: application/http - * Content-ID: 1 - * - * POST /v1/records:queryRecord?key=${key} - * Content-Type: application/json - * Accept: application/json - * - * { - * "origin":"https://github.com" - * } - * - * --BATCH_BOUNDARY-- - * - * @param {BatchValues} batchValues - * @param {string} key - */ - -function generateBatchBody(batchValues, key) { - let str = '' - batchValues.forEach(({ options, result }, index) => { - if (result !== undefined) return - str += `--${boundary} -Content-Type: application/http -Content-ID: ${index + 1} - -POST /v1/records:queryRecord?key=${key} -Content-Type: application/json -Accept: application/json - -${JSON.stringify(options, null, ' ')} - -` - }) - return `${str}\n--${boundary}--` -} - -/** - * Naive multipart parser: - * - use Content-ID to find the response id - * - use "{" as a start of a body and "}" as an end - * - * `text` example: - * - * --batch_acyIJf8nRW5t11AyZUOwieHC_eWk1alw - * Content-Type: application/http - * Content-ID: response-1 - * - * HTTP/1.1 200 OK - * Content-Type: application/json; charset=UTF-8 - * Vary: Origin - * Vary: X-Origin - * Vary: Referer - * - * { - * "record": { - * "key": { - * "origin": "https://example.com" - * }, - * "metrics": { - * "cumulative_layout_shift": { - * "histogram": [ - * { - * "start": "0.00", - * "end": "0.10", - * "density": 0.98919891989199 - * }, - * { - * "start": "0.10", - * "end": "0.25", - * "density": 0.0034003400340034034 - * }, - * { - * "start": "0.25", - * "density": 0.00740074007400741 - * } - * ], - * "percentiles": { - * "p75": "0.04" - * } - * }, - * "first_contentful_paint": { - * "histogram": [ - * { - * "start": 0, - * "end": 1000, - * "density": 0.37636345441809338 - * }, - * { - * "start": 1000, - * "end": 3000, - * "density": 0.4767337135995206 - * }, - * { - * "start": 3000, - * "density": 0.14690283198238688 - * } - * ], - * "percentiles": { - * "p75": 2207 - * } - * }, - * "first_input_delay": { - * "histogram": [ - * { - * "start": 0, - * "end": 100, - * "density": 0.9704911473442015 - * }, - * { - * "start": 100, - * "end": 300, - * "density": 0.020806241872561727 - * }, - * { - * "start": 300, - * "density": 0.0087026107832349486 - * } - * ], - * "percentiles": { - * "p75": 21 - * } - * }, - * "largest_contentful_paint": { - * "histogram": [ - * { - * "start": 0, - * "end": 2500, - * "density": 0.79676870748299278 - * }, - * { - * "start": 2500, - * "end": 4000, - * "density": 0.1136954781912765 - * }, - * { - * "start": 4000, - * "density": 0.089535814325730309 - * } - * ], - * "percentiles": { - * "p75": 2215 - * } - * } - * } - * } - * } - * - * --batch_acyIJf8nRW5t11AyZUOwieHC_eWk1alw - * Content-Type: application/http - * Content-ID: response-2 - * - * HTTP/1.1 404 Not Found - * Vary: Origin - * Vary: X-Origin - * Vary: Referer - * Content-Type: application/json; charset=UTF-8 - * - * { - * "error": { - * "code": 404, - * "message": "chrome ux report data not found", - * "status": "NOT_FOUND" - * } - * } - * - * --batch_acyIJf8nRW5t11AyZUOwieHC_eWk1alw-- - * - * @param {string} text - */ - -function parseBatchResponse(text) { - const results = /** @type {{ index: number, json: any }[]} */ ([]) - let index = /** @type {number | null} */ (null) - let contentBody = '' - for (const line of text.split('\n')) { - if (line.startsWith('Content-ID')) { - const m = line.match(/response\-(\d*)/) || [] - index = parseInt(m[1]) - } - if ((index && line.startsWith('{')) || contentBody) { - contentBody += line - } - if (index && contentBody && line.startsWith('}')) { - results.push({ index: index - 1, json: JSON.parse(contentBody) }) - index = null - contentBody = '' - } - } - return results -} diff --git a/package.json b/package.json index c570932..33c981d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "crux-api", - "version": "1.1.1", - "description": "A Chrome UX Report API wrapper wrapper that supports batching, handles errors, and provides types.", + "version": "2.0.0", + "description": "A Chrome UX Report API wrapper wrapper that handles errors and provides types", "repository": "https://github.com/treosh/crux-api", "bugs": "https://github.com/treosh/crux-api/issues", "license": "MIT", @@ -10,7 +10,6 @@ "types": "dist/crux-api.d.ts", "main": "dist/crux-api.js", "files": [ - "batch", "dist", "src" ], @@ -18,9 +17,7 @@ "build": "run-s build:*", "build:src": "microbundle build --no-sourcemap --format=cjs", "build:src-ts": "tsc --declaration --noEmit false --outDir dist/ --allowJs src/index.js && rm dist/index.js && mv dist/index.d.ts dist/crux-api.d.ts", - "build:batch": "rm -rf batch/dist && microbundle build --no-sourcemap --format=cjs --cwd batch", - "build:batch-ts": "tsc --declaration --noEmit false --outDir batch/dist --allowJs batch/src/index.js && mv batch/dist/batch/src/index.d.ts batch/dist/crux-api-batch.d.ts && rm -rf batch/dist/{src,batch}", - "test": "ava -v && prettier -c src batch/src test script README.md && yarn build && tsc -p . && size-limit", + "test": "ava -v && prettier -c src test script README.md && yarn build && tsc -p . && size-limit", "prepack": "yarn build" }, "devDependencies": { @@ -38,7 +35,6 @@ "keywords": [ "CrUX", "CrUX API", - "Batch API", "Chrome User Experience Report", "Chrome UX Report", "Core Web Vitals", @@ -57,10 +53,6 @@ { "limit": "450B", "path": "./src/index.js" - }, - { - "limit": "850B", - "path": "./batch/src/index.js" } ], "ava": { diff --git a/script/batch-limits.js b/script/batch-limits.js deleted file mode 100644 index 0df9c45..0000000 --- a/script/batch-limits.js +++ /dev/null @@ -1,102 +0,0 @@ -// A script to test batch API limits using Github's URLs & origins. -// Script generates 312 + 720 requests, which quicly exaust CrUX API limits for the `key`. -// -// Check the data using the node CLI: -// -// > o = require('./result/origins3.json') -// > o.length -// 312 -// > new Set(o.filter(v => v.error).map(v => v.error.code)) -// Set { 429, 404 } -// -// usage: CRUX_KEY='...' node -r esm script/batch-limits.js 1 - -import nodeFetch from 'node-fetch' -import { promises as fs } from 'fs' -import { join } from 'path' -import { createBatch } from '../batch/src' - -const key = process.env.CRUX_KEY || 'no-key' -const suffix = process.argv[2] || '' // the suffix for `result/${origins|urls}${suffix}.json` file -const origins = [ - 'https://github.com', - 'https://resources.github.com', - 'https://developer.github.com', - 'https://atom.io', - 'https://www.electronjs.org', - 'https://desktop.github.com', - 'https://partner.github.com', - 'https://docs.github.com', - 'https://www.githubstatus.com', - 'https://support.github.com', - 'https://github.myshopify.com', - 'https://socialimpact.github.com', - 'https://github.blog', -] -const urls = [ - 'https://github.com/', - 'https://github.com/team', - 'https://github.com/enterprise', - 'https://github.com/marketplace', - 'https://github.com/pricing', - 'https://github.com/explore', - 'https://github.com/login?return_to=%2Fexplore', - 'https://github.com/join?ref_cta=Sign+up&ref_loc=header+logged+out&ref_page=%2Fexplore&source=header', - 'https://github.com/features', - 'https://github.com/features/actions', - 'https://github.com/features/code-review/', - 'https://github.com/features/project-management/', - 'https://github.com/security', - 'https://github.com/customer-stories?type=enterprise', - 'https://github.com/about', - 'https://github.com/about/careers', - 'https://github.com/about/press', - 'https://resources.github.com/', - 'https://developer.github.com/', - 'https://atom.io/', - 'https://www.electronjs.org/', - 'https://desktop.github.com/', - 'https://partner.github.com/', - 'https://docs.github.com/', - 'https://www.githubstatus.com/', - 'https://support.github.com/', - 'https://github.myshopify.com/', - 'https://socialimpact.github.com/', - 'https://github.blog/', - 'https://github.blog/changelog/', -] -const formFactors = /** @type {import('../src').FormFactor[]} */ ([undefined, 'PHONE', 'DESKTOP', 'TABLET']) -const connections = /** @type {import('../src').Connection[]} */ ([undefined, '4G', '3G', '2G', 'slow-2G', 'offline']) - -async function main() { - const batch = createBatch({ key, fetch: nodeFetch }) - console.time('fetch origins') - const res1 = await batch( - origins.reduce((memo, origin) => { - for (const formFactor of formFactors) { - for (const effectiveConnectionType of connections) { - memo.push({ origin, formFactor, effectiveConnectionType }) - } - } - return memo - }, /** @type {import('../src').BatchOptions} */ ([])) - ) - await fs.writeFile(join(__dirname, `../result/origins${suffix}.json`), JSON.stringify(res1, null, ' ')) - console.timeEnd('fetch origins') - - console.time('fetch urls') - const res2 = await batch( - urls.reduce((memo, url) => { - for (const formFactor of formFactors) { - for (const effectiveConnectionType of connections) { - memo.push({ url, formFactor, effectiveConnectionType }) - } - } - return memo - }, /** @type {import('../src').BatchOptions} */ ([])) - ) - await fs.writeFile(join(__dirname, `../result/urls${suffix}.json`), JSON.stringify(res2, null, ' ')) - console.timeEnd('fetch urls') -} - -main().catch(console.error) diff --git a/script/batch.js b/script/batch.js deleted file mode 100644 index a9dd0ba..0000000 --- a/script/batch.js +++ /dev/null @@ -1,17 +0,0 @@ -// usage: CRUX_KEY='...' node -r esm script/batch.js -import nodeFetch from 'node-fetch' -import { createBatch } from '../batch/src' - -const key = process.env.CRUX_KEY || 'no-key' - -async function main() { - const batch = createBatch({ key, fetch: nodeFetch }) - const res = await batch([ - { origin: 'https://example.com' }, - { url: 'https://github.com/', formFactor: 'DESKTOP' }, - { origin: 'https://fooo.bar' }, - ]) - console.log(JSON.stringify(res, null, ' ')) -} - -main().catch(console.error) diff --git a/src/index.js b/src/index.js index aa49933..d63e105 100644 --- a/src/index.js +++ b/src/index.js @@ -1,11 +1,9 @@ -import { retryAfterTimeout } from './retry' +const maxRetries = 10 +const maxRetryTimeout = 60 * 1000 // 60s /** * @typedef {{ key: string, fetch?: function }} CreateOptions * @typedef {{ url?: string, origin?: string, formFactor?: FormFactor, effectiveConnectionType?: Connection }} QueryRecordOptions - * @typedef {QueryRecordOptions[]} BatchOptions - * @typedef {(SuccessResponse | null)[]} BatchResponse - * * @typedef {'ALL_FORM_FACTORS' | 'PHONE' | 'DESKTOP' | 'TABLET'} FormFactor * @typedef {'4G' | '3G' | '2G' | 'slow-2G' | 'offline'} Connection * @typedef {{ histogram: { start: number | string, end: number | string, density: number }[], percentiles: { p75: number | string } }} MetricValue @@ -77,3 +75,21 @@ export function normalizeUrl(url) { const u = new URL(url) return u.origin + u.pathname } + +/** + * Random delay from 1ms to `maxRetryTimeout`. + * Random logic is based on: https://stackoverflow.com/a/29246176 + * + * @param {number} retryCounter + * @param {function} request + */ + +export async function retryAfterTimeout(retryCounter, request) { + if (retryCounter <= maxRetries) { + const timeout = Math.floor(Math.random() * maxRetryTimeout) + 1 + await new Promise((resolve) => setTimeout(resolve, timeout)) + return request() + } else { + throw new Error('Max retries reached') + } +} diff --git a/src/retry.js b/src/retry.js deleted file mode 100644 index 1ca7b15..0000000 --- a/src/retry.js +++ /dev/null @@ -1,20 +0,0 @@ -const maxRetries = 10 -const maxRetryTimeout = 100 * 1000 // 100s - -/** - * Random delay from 1ms to `maxRetryTimeout`. - * Random logic is based on: https://stackoverflow.com/a/29246176 - * - * @param {number} retryCounter - * @param {function} request - */ - -export async function retryAfterTimeout(retryCounter, request) { - if (retryCounter <= maxRetries) { - const timeout = Math.floor(Math.random() * maxRetryTimeout) + 1 - await new Promise((resolve) => setTimeout(resolve, timeout)) - return request() - } else { - throw new Error('Max retries reached') - } -} diff --git a/test/batch.js b/test/batch.js deleted file mode 100644 index b2a31b2..0000000 --- a/test/batch.js +++ /dev/null @@ -1,23 +0,0 @@ -// yarn ava test/batch.js -import test from 'ava' -import fetch from 'node-fetch' -import { createBatch } from '../batch/src' - -const key = process.env.CRUX_KEY || 'no-key' - -test('createBatch', async (t) => { - const batch = createBatch({ key, fetch }) - const results1 = await batch([ - { origin: 'https://github.com', formFactor: 'DESKTOP' }, - { url: 'https://github.com/explore', effectiveConnectionType: '4G' }, - ]) - t.is(results1.length, 2) - - const r1 = /** @type {import('../src').SuccessResponse} */ (results1[0]) - t.is(r1.record.key.origin, 'https://github.com') - t.deepEqual(Object.keys(r1.record.key), ['formFactor', 'origin']) - - const r2 = /** @type {import('../src').SuccessResponse} */ (results1[1]) - t.is(r2.record.key.url, 'https://github.com/explore') - t.deepEqual(Object.keys(r2.record.key), ['effectiveConnectionType', 'url']) -}) diff --git a/test/index.js b/test/index.js index 9a24cff..41e1908 100644 --- a/test/index.js +++ b/test/index.js @@ -26,7 +26,7 @@ test('normalizeUrl', async (t) => { const queryRecord = createQueryRecord({ key, fetch }) const urls = [ ['https://www.gov.uk', 'https://www.gov.uk/'], // adds / - ['https://hey.com/features/', 'https://hey.com/features/'], // no change, URL with / + ['https://www.hey.com/features/', 'https://www.hey.com/features/'], // no change, URL with / ['https://stripe.com/docs/api', 'https://stripe.com/docs/api'], // no change, URL without / ['https://github.com/marketplace?type=actions', 'https://github.com/marketplace'], // removes search params ] diff --git a/tsconfig.json b/tsconfig.json index 854d1d9..5f2c551 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,5 +10,5 @@ "noUnusedLocals": true, "noUnusedParameters": true }, - "include": ["batch/src", "src", "test", "script"] + "include": ["src", "test", "script"] }