From 15691a402709abd2526aaf13f99176e39e4219ee Mon Sep 17 00:00:00 2001
From: fraxken
Date: Sat, 6 Jul 2024 14:44:48 +0200
Subject: [PATCH 1/2] feat: implement ManifestManager workspace & refactor
tarball
---
README.md | 1 +
package-lock.json | 57 +-
package.json | 5 +-
tsconfig.json | 3 +
workspaces/conformance/package.json | 2 +-
workspaces/mama/README.md | 144 +++++
workspaces/mama/package.json | 39 ++
workspaces/mama/src/index.ts | 158 +++++
workspaces/mama/test/ManifestManager.spec.ts | 573 ++++++++++++++++++
workspaces/mama/tsconfig.json | 13 +
workspaces/npm-types/README.md | 2 +-
workspaces/npm-types/src/index.d.ts | 38 +-
workspaces/scanner/test/depWalker.spec.ts | 1 +
.../fixtures/depWalker/slimio.is-result.json | 2 +-
workspaces/tarball/README.md | 2 +-
workspaces/tarball/package.json | 5 +-
workspaces/tarball/src/constants.ts | 14 -
workspaces/tarball/src/manifest.ts | 85 ---
workspaces/tarball/src/tarball.ts | 104 ++--
workspaces/tarball/src/types.ts | 32 -
.../tarball/src/utils/analyzeDependencies.ts | 135 +++--
workspaces/tarball/test/manifest.spec.ts | 137 -----
.../test/utils/analyzeDependencies.spec.ts | 110 ++--
workspaces/tarball/tsconfig.json | 3 +
workspaces/tree-walker/README.md | 2 +-
25 files changed, 1239 insertions(+), 428 deletions(-)
create mode 100644 workspaces/mama/README.md
create mode 100644 workspaces/mama/package.json
create mode 100644 workspaces/mama/src/index.ts
create mode 100644 workspaces/mama/test/ManifestManager.spec.ts
create mode 100644 workspaces/mama/tsconfig.json
delete mode 100644 workspaces/tarball/src/constants.ts
delete mode 100644 workspaces/tarball/src/manifest.ts
delete mode 100644 workspaces/tarball/src/types.ts
delete mode 100644 workspaces/tarball/test/manifest.spec.ts
diff --git a/README.md b/README.md
index e53da54..e12e3f1 100644
--- a/README.md
+++ b/README.md
@@ -132,6 +132,7 @@ Click on one of the links to access the documentation of the workspace:
| --- | --- |
| tarball | [@nodesecure/sec-literal](./workspaces/tarball) |
| tree-walker | [@nodesecure/tree-walker](./workspaces/tree-walker) |
+| mama | [@nodesecure/mama](./workspaces/mama) |
| conformance | [@nodesecure/npm-types](./workspaces/conformance) |
| npm-types | [@nodesecure/npm-types](./workspaces/npm-types) |
diff --git a/package-lock.json b/package-lock.json
index 4168c30..5ad7e55 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,16 +1,15 @@
{
- "name": "@nodesecure/scanner",
+ "name": "scanner",
"version": "5.3.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
- "name": "@nodesecure/scanner",
- "version": "5.3.0",
"license": "MIT",
"workspaces": [
"workspaces/scanner",
"workspaces/tarball",
+ "workspaces/mama",
"workspaces/tree-walker",
"workspaces/conformance",
"workspaces/npm-types"
@@ -1686,6 +1685,10 @@
"node": ">=18.0.0"
}
},
+ "node_modules/@nodesecure/mama": {
+ "resolved": "workspaces/mama",
+ "link": true
+ },
"node_modules/@nodesecure/npm-registry-sdk": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/@nodesecure/npm-registry-sdk/-/npm-registry-sdk-3.0.0.tgz",
@@ -3523,19 +3526,6 @@
"integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==",
"dev": true
},
- "node_modules/@types/lodash": {
- "version": "4.17.6",
- "dev": true,
- "license": "MIT"
- },
- "node_modules/@types/lodash.difference": {
- "version": "4.5.9",
- "dev": true,
- "license": "MIT",
- "dependencies": {
- "@types/lodash": "*"
- }
- },
"node_modules/@types/minimist": {
"version": "1.2.2",
"dev": true,
@@ -3606,6 +3596,12 @@
"@types/node": "*"
}
},
+ "node_modules/@types/object-hash": {
+ "version": "3.0.6",
+ "resolved": "https://registry.npmjs.org/@types/object-hash/-/object-hash-3.0.6.tgz",
+ "integrity": "sha512-fOBV8C1FIu2ELinoILQ+ApxcUKz4ngq+IWUYrxSGjXzzjUALijilampwkMgEtJ+h2njAW3pi853QpzNVCHB73w==",
+ "dev": true
+ },
"node_modules/@types/pacote": {
"version": "11.1.8",
"dev": true,
@@ -6001,10 +5997,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
- "node_modules/lodash.difference": {
- "version": "4.5.0",
- "license": "MIT"
- },
"node_modules/lodash.get": {
"version": "4.4.2",
"license": "MIT"
@@ -7054,6 +7046,14 @@
"encoding": "^0.1.13"
}
},
+ "node_modules/object-hash": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz",
+ "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==",
+ "engines": {
+ "node": ">= 6"
+ }
+ },
"node_modules/once": {
"version": "1.4.0",
"license": "ISC",
@@ -9061,7 +9061,7 @@
"node-estree": "^4.0.0"
},
"engines": {
- "node": "=>20"
+ "node": ">=20"
}
},
"workspaces/conformance/node_modules/@myunisoft/httpie": {
@@ -9107,6 +9107,17 @@
"node": ">=18.17"
}
},
+ "workspaces/mama": {
+ "name": "@nodesecure/mama",
+ "version": "1.0.0",
+ "license": "MIT",
+ "dependencies": {
+ "object-hash": "^3.0.0"
+ },
+ "devDependencies": {
+ "@types/object-hash": "^3.0.6"
+ }
+ },
"workspaces/npm-types": {
"name": "@nodesecure/npm-types",
"version": "1.0.0",
@@ -9146,14 +9157,12 @@
"@nodesecure/conformance": "^1.0.0",
"@nodesecure/fs-walk": "^2.0.0",
"@nodesecure/js-x-ray": "^6.3.0",
+ "@nodesecure/mama": "^1.0.0",
"@nodesecure/npm-types": "^1.0.0",
"@nodesecure/utils": "^2.1.0",
- "builtins": "^5.1.0",
- "lodash.difference": "^4.5.0",
"pacote": "^18.0.6"
},
"devDependencies": {
- "@types/lodash.difference": "^4.5.9",
"get-folder-size": "^4.0.0"
}
},
diff --git a/package.json b/package.json
index 5ce18a1..02b2219 100644
--- a/package.json
+++ b/package.json
@@ -1,11 +1,8 @@
{
- "name": "@nodesecure/scanner",
- "version": "5.3.0",
- "description": "A package API to run a static analysis of your module's dependencies.",
- "type": "module",
"workspaces": [
"workspaces/scanner",
"workspaces/tarball",
+ "workspaces/mama",
"workspaces/tree-walker",
"workspaces/conformance",
"workspaces/npm-types"
diff --git a/tsconfig.json b/tsconfig.json
index 49a40a7..9ef045f 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -7,6 +7,9 @@
{
"path": "./workspaces/conformance"
},
+ {
+ "path": "./workspaces/mama"
+ },
{
"path": "./workspaces/tarball"
},
diff --git a/workspaces/conformance/package.json b/workspaces/conformance/package.json
index d19d6f3..39d4844 100644
--- a/workspaces/conformance/package.json
+++ b/workspaces/conformance/package.json
@@ -6,7 +6,7 @@
"exports": "./dist/index.js",
"types": "./dist/index.d.ts",
"engines": {
- "node": "=>20"
+ "node": ">=20"
},
"scripts": {
"build": "tsc -b",
diff --git a/workspaces/mama/README.md b/workspaces/mama/README.md
new file mode 100644
index 0000000..44ab771
--- /dev/null
+++ b/workspaces/mama/README.md
@@ -0,0 +1,144 @@
+
+ @nodesecure/mama
+
+
+
+ Manifest Manager
+
+
+## Requirements
+- [Node.js](https://nodejs.org/en/) v20 or higher
+
+## Getting Started
+
+This package is available in the Node Package Repository and can be easily installed with [npm](https://docs.npmjs.com/getting-started/what-is-npm) or [yarn](https://yarnpkg.com).
+
+```bash
+$ npm i @nodesecure/mama
+# or
+$ yarn add @nodesecure/mama
+```
+
+## Usage example
+
+```ts
+import { ManifestManager } from "@nodesecure/mama";
+
+const mama = await ManifestManager.fromPackageJSON(
+ process.cwd()
+);
+console.log(mama.document);
+console.log(mama.integrity);
+```
+
+## API
+
+### (static) fromPackageJSON(location: string): Promise< ManifestManager >
+
+Load a new instance using a `package.json` from the filesystem.
+
+The **location** parameter can either be a full path or the path to the directory where the `package.json` is located.
+
+### constructor(document: ManifestManagerDocument)
+
+```ts
+type ManifestManagerDocument =
+ PackageJSON |
+ WorkspacesPackageJSON |
+ PackumentVersion;
+```
+
+Default values are injected if they are not present in the document. This behavior is necessary for the correct operation of certain functions, such as integrity recovery.
+
+```js
+{
+ dependencies: {},
+ devDependencies: {},
+ scripts: {},
+ gypfile: false
+}
+```
+
+> [!NOTE]
+> document is deep cloned (there will no longer be any reference to the object supplied as an argument)
+
+### spec
+Return the NPM specification (which is the combinaison of `name@version`).
+
+> [!CAUTION]
+> This property may not be available for Workspaces (if 'name' or 'version' properties are missing, it will throw an error).
+
+### integrity
+Return an integrity hash (which is a **string**) of the following properties:
+
+```js
+{
+ name,
+ version,
+ dependencies,
+ license: license ?? "NONE",
+ scripts
+}
+```
+
+If `dependencies` and `scripts` are missing, they are defaulted to an empty object `{}`
+
+> [!CAUTION]
+> This is not available for Workspaces
+
+### author
+Return the author parsed as a **Contact** (or `null` if the property is missing).
+
+```ts
+interface Contact {
+ email?: string;
+ url?: string;
+ name: string;
+}
+```
+
+### dependencies and devDependencies
+Return the (dev) dependencies as an Array (of string)
+
+```json
+{
+ "dependencies": {
+ "kleur": "1.0.0"
+ }
+}
+```
+
+The above JSON will produce `["kleur"]`
+
+### isWorkspace
+Return true if `workspaces` property is present
+
+> [!NOTE]
+> Workspace are described by the interface `WorkspacesPackageJSON` (from @nodesecure/npm-types)
+
+### flags
+
+Since we've created this package for security purposes, the instance contains various flags indicating threats detected in the content:
+
+- **isNative**: Contain an identified native package to build or provide N-API features like `node-gyp`.
+- **hasUnsafeScripts**: Contain unsafe scripts like `install`, `preinstall`, `postinstall`...
+
+```ts
+import assert from "node:assert";
+
+const mama = new ManifestManager({
+ name: "hello",
+ version: "1.0.0",
+ scripts: {
+ postinstall: "node malicious.js"
+ }
+});
+
+assert.ok(mama.flags.hasUnsafeScripts);
+```
+
+The flags property is sealed (It is not possible to extend the list of flags)
+
+> [!IMPORTANT]
+> Read more about unsafe scripts [here](https://www.nerdycode.com/prevent-npm-executing-scripts-security/)
+
diff --git a/workspaces/mama/package.json b/workspaces/mama/package.json
new file mode 100644
index 0000000..5edadf0
--- /dev/null
+++ b/workspaces/mama/package.json
@@ -0,0 +1,39 @@
+{
+ "name": "@nodesecure/mama",
+ "version": "1.0.0",
+ "description": "Manifest Manager",
+ "type": "module",
+ "exports": "./dist/index.js",
+ "types": "./dist/index.d.ts",
+ "scripts": {
+ "build": "tsc -b",
+ "prepublishOnly": "npm run build",
+ "test-only": "glob -c \"tsx --test\" \"./test/**/*.spec.ts\"",
+ "test": "c8 -r html npm run test-only"
+ },
+ "files": [
+ "dist"
+ ],
+ "keywords": [
+ "manifest",
+ "manager",
+ "pacote",
+ "security"
+ ],
+ "author": "GENTILHOMME Thomas ",
+ "license": "MIT",
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/NodeSecure/scanner.git"
+ },
+ "bugs": {
+ "url": "https://github.com/NodeSecure/scanner/issues"
+ },
+ "homepage": "https://github.com/NodeSecure/tree/master/workspaces/mama#readme",
+ "dependencies": {
+ "object-hash": "^3.0.0"
+ },
+ "devDependencies": {
+ "@types/object-hash": "^3.0.6"
+ }
+}
diff --git a/workspaces/mama/src/index.ts b/workspaces/mama/src/index.ts
new file mode 100644
index 0000000..5a3ce3d
--- /dev/null
+++ b/workspaces/mama/src/index.ts
@@ -0,0 +1,158 @@
+// Import Node.js Dependencies
+import fs from "node:fs/promises";
+import path from "node:path";
+
+// Import Third-party Dependencies
+import hash from "object-hash";
+import { parseAuthor } from "@nodesecure/utils";
+import type {
+ PackumentVersion, PackageJSON, WorkspacesPackageJSON, Contact
+} from "@nodesecure/npm-types";
+
+type WithRequired = T & { [P in K]-?: T[P] }
+
+export type NonOptionalPackageJSONProperties =
+ "dependencies" |
+ "devDependencies" |
+ "scripts" |
+ "gypfile";
+
+// CONSTANTS
+const kNativeNpmPackages = new Set([
+ "node-gyp",
+ "node-pre-gyp",
+ "node-gyp-build",
+ "node-addon-api"
+]);
+
+/**
+ * @see https://www.nerdycode.com/prevent-npm-executing-scripts-security/
+ */
+export const kUnsafeNPMScripts = new Set([
+ "install",
+ "preinstall",
+ "postinstall",
+ "preuninstall",
+ "postuninstall"
+]);
+
+export type ManifestManagerDefaultProperties = Required<
+ Pick
+>;
+
+export type ManifestManagerDocument =
+ PackageJSON |
+ WorkspacesPackageJSON |
+ PackumentVersion;
+
+export class ManifestManager<
+ MetadataDef extends Record = Record
+> {
+ static Default: Readonly = Object.freeze({
+ dependencies: {},
+ devDependencies: {},
+ scripts: {},
+ gypfile: false
+ });
+
+ public metadata: MetadataDef = Object.create(null);
+ public document: WithRequired<
+ ManifestManagerDocument,
+ NonOptionalPackageJSONProperties
+ >;
+
+ public flags = Object.seal({
+ hasUnsafeScripts: false,
+ isNative: false
+ });
+
+ constructor(
+ document: ManifestManagerDocument
+ ) {
+ this.document = Object.assign(
+ { ...ManifestManager.Default },
+ structuredClone(document)
+ );
+
+ this.flags.isNative = [
+ ...this.dependencies,
+ ...this.devDependencies
+ ].some((pkg) => kNativeNpmPackages.has(pkg)) || this.document.gypfile;
+ this.flags.hasUnsafeScripts = Object
+ .keys(this.document.scripts)
+ .some((script) => kUnsafeNPMScripts.has(script.toLowerCase()));
+ }
+
+ get nodejsImports() {
+ return this.document.imports ?? {};
+ }
+
+ get dependencies() {
+ return Object.keys(this.document.dependencies);
+ }
+
+ get devDependencies() {
+ return Object.keys(this.document.devDependencies);
+ }
+
+ get spec(): `${string}@${string}` {
+ const hasBothProperties = ["name", "version"]
+ .every((key) => key in this.document);
+ if (this.isWorkspace && !hasBothProperties) {
+ throw new Error("spec is not available for the given workspace");
+ }
+
+ return `${this.document.name}@${this.document.version}`;
+ }
+
+ get author(): Contact | null {
+ return parseAuthor(this.document.author);
+ }
+
+ get isWorkspace(): boolean {
+ return "workspaces" in this.document;
+ }
+
+ get integrity(): string {
+ if (this.isWorkspace) {
+ throw new Error("integrity is not available for workspaces");
+ }
+
+ return hash({
+ name: this.document.name,
+ version: this.document.version,
+ dependencies: this.document.dependencies,
+ license: this.document.license ?? "NONE",
+ scripts: this.document.scripts
+ });
+ }
+
+ static async fromPackageJSON(
+ location: string
+ ): Promise {
+ if (typeof location !== "string") {
+ throw new TypeError("location must be a string primitive");
+ }
+
+ const packageLocation = location.endsWith("package.json") ?
+ location :
+ path.join(location, "package.json");
+ const packageStr = await fs.readFile(packageLocation, "utf-8");
+
+ try {
+ const packageJSON = JSON.parse(
+ packageStr
+ ) as PackageJSON | WorkspacesPackageJSON;
+
+ return new ManifestManager(
+ packageJSON
+ );
+ }
+ catch (cause) {
+ throw new Error(
+ `Failed to parse package.json located at: ${packageLocation}`,
+ { cause }
+ );
+ }
+ }
+}
diff --git a/workspaces/mama/test/ManifestManager.spec.ts b/workspaces/mama/test/ManifestManager.spec.ts
new file mode 100644
index 0000000..97f4bf1
--- /dev/null
+++ b/workspaces/mama/test/ManifestManager.spec.ts
@@ -0,0 +1,573 @@
+// Import Node.js Dependencies
+import fs from "node:fs/promises";
+import path from "node:path";
+import assert from "node:assert";
+import { describe, it, test } from "node:test";
+
+// Import Third-party Dependencies
+import type {
+ PackageJSON,
+ WorkspacesPackageJSON
+} from "@nodesecure/npm-types";
+import hash from "object-hash";
+import sinon from "sinon";
+
+// Import Internal Dependencies
+import { ManifestManager } from "../src/index.js";
+
+// CONSTANTS
+const kMinimalPackageJSON = {
+ name: "foo",
+ version: "1.5.0"
+};
+const kMinimalPackageJSONIntegrity = hash({
+ ...kMinimalPackageJSON,
+ dependencies: {},
+ scripts: {},
+ license: "NONE"
+});
+
+describe("ManifestManager", () => {
+ describe("static Default", () => {
+ test("Property must be Frozen", () => {
+ const isUpdated = Reflect.set(ManifestManager.Default, "foo", "bar");
+
+ assert.strictEqual(isUpdated, false);
+ assert.ok(Object.isFrozen(ManifestManager.Default));
+ });
+ });
+
+ describe("static fromPackageJSON()", () => {
+ test(`Given a location equal to process.cwd(),
+ it should read and parse the JSON from filesystem`, async() => {
+ const readFile = sinon
+ .stub(fs, "readFile")
+ .resolves(JSON.stringify(kMinimalPackageJSON));
+
+ try {
+ const location = process.cwd();
+ const mama = await ManifestManager.fromPackageJSON(
+ location
+ );
+
+ assert.ok(readFile.calledOnce);
+ assert.ok(mama instanceof ManifestManager);
+
+ assert.strictEqual(
+ mama.isWorkspace,
+ false
+ );
+ assert.strictEqual(
+ mama.spec,
+ `${kMinimalPackageJSON.name}@${kMinimalPackageJSON.version}`
+ );
+ }
+ finally {
+ readFile.restore();
+ }
+ });
+
+ test(`Given a location equal to process.cwd() + package.json,
+ it should read and parse the JSON from filesystem`, async() => {
+ const readFile = sinon
+ .stub(fs, "readFile")
+ .resolves(JSON.stringify(kMinimalPackageJSON));
+
+ try {
+ const location = path.join(process.cwd(), "package.json");
+ const mama = await ManifestManager.fromPackageJSON(
+ location
+ );
+
+ assert.ok(readFile.calledOnce);
+ assert.ok(mama instanceof ManifestManager);
+
+ assert.strictEqual(
+ mama.isWorkspace,
+ false
+ );
+ assert.strictEqual(
+ mama.spec,
+ `${kMinimalPackageJSON.name}@${kMinimalPackageJSON.version}`
+ );
+ }
+ finally {
+ readFile.restore();
+ }
+ });
+
+ test("Given an invalid JSON, it should throw a custom Error with the parsing error as a cause", async(t) => {
+ const location = process.cwd();
+ const expectedLocation = path.join(location, "package.json");
+
+ const readFile = sinon
+ .stub(fs, "readFile")
+ .resolves(`{ foo: NaN }`);
+ t.after(() => readFile.restore());
+ // TODO: add t.plan(5) when available in LTS version of test_runner
+
+ try {
+ await ManifestManager.fromPackageJSON(
+ process.cwd()
+ );
+ }
+ catch (error) {
+ assert.strictEqual(error.name, "Error");
+ assert.strictEqual(
+ error.message,
+ `Failed to parse package.json located at: ${expectedLocation}`
+ );
+
+ assert.ok("cause" in error);
+ assert.strictEqual(error.cause.name, "SyntaxError");
+ }
+
+ assert.ok(readFile.calledOnce);
+ });
+
+ test("Given the location argument is not a string, it must throw a TypeError", async() => {
+ await assert.rejects(
+ () => ManifestManager.fromPackageJSON({} as any),
+ {
+ name: "TypeError",
+ message: "location must be a string primitive"
+ }
+ );
+ });
+ });
+
+ describe("constructor()", () => {
+ it("Should deep clone the provided document", () => {
+ const mama = new ManifestManager(kMinimalPackageJSON);
+
+ assert.notStrictEqual(mama.document, kMinimalPackageJSON);
+ });
+
+ it("Should set default values for multiple properties if they are not present in the provided document.", () => {
+ const mama = new ManifestManager(kMinimalPackageJSON);
+
+ assert.deepStrictEqual(
+ mama.document,
+ {
+ ...kMinimalPackageJSON,
+ ...ManifestManager.Default
+ }
+ )
+ });
+ });
+
+ describe("get dependencies", () => {
+ test("Given a PackageJSON with no dependencies, it must return an empty Array", () => {
+ const packageJSON: PackageJSON = {
+ ...kMinimalPackageJSON
+ };
+
+ const mama = new ManifestManager(packageJSON);
+ assert.deepEqual(mama.dependencies, []);
+ });
+
+ test("Given a PackageJSON with multiple dependencies, it must return their names", () => {
+ const packageJSON: PackageJSON = {
+ ...kMinimalPackageJSON,
+ dependencies: {
+ kleur: "1.0.0",
+ mocha: "1.0.0"
+ }
+ };
+
+ const mama = new ManifestManager(packageJSON);
+ assert.deepEqual(
+ mama.dependencies,
+ ["kleur", "mocha"]
+ );
+ });
+ });
+
+ describe("get devDependencies", () => {
+ test("Given a PackageJSON with no devDependencies, it must return an empty Array", () => {
+ const packageJSON: PackageJSON = {
+ ...kMinimalPackageJSON
+ };
+
+ const mama = new ManifestManager(packageJSON);
+ assert.deepEqual(mama.devDependencies, []);
+ });
+
+ test("Given a PackageJSON with multiple devDependencies, it must return their names", () => {
+ const packageJSON: PackageJSON = {
+ ...kMinimalPackageJSON,
+ devDependencies: {
+ kleur: "1.0.0",
+ mocha: "1.0.0"
+ }
+ };
+
+ const mama = new ManifestManager(packageJSON);
+ assert.deepEqual(
+ mama.devDependencies,
+ ["kleur", "mocha"]
+ );
+ });
+ });
+
+ describe("get nodejsImports", () => {
+ test("Given a PackageJSON with no imports, it must return an empty Object", () => {
+ const packageJSON: PackageJSON = {
+ ...kMinimalPackageJSON
+ };
+
+ const mama = new ManifestManager(packageJSON);
+ assert.deepStrictEqual(mama.nodejsImports, {});
+ });
+
+ test("Given a PackageJSON with one import subpath, it must be returned unchanged", () => {
+ const nodeImport: PackageJSON["imports"] = {
+ "#dep": {
+ "node": "something"
+ }
+ };
+ const packageJSON: PackageJSON = {
+ ...kMinimalPackageJSON,
+ imports: { ...nodeImport }
+ };
+
+ const mama = new ManifestManager(packageJSON);
+ assert.deepStrictEqual(
+ mama.nodejsImports,
+ nodeImport
+ );
+ });
+ });
+
+ describe("get spec", () => {
+ test("Given a PackageJSON, it should return the NPM spec by combining the name and version", () => {
+ const packageJSON: PackageJSON = {
+ ...kMinimalPackageJSON
+ };
+
+ const mama = new ManifestManager(packageJSON);
+
+ assert.strictEqual(
+ mama.spec,
+ `${packageJSON.name}@${packageJSON.version}`
+ );
+ });
+
+ test(`Given a WorkspacesPackageJSON with 'name' and 'version' properties available,
+ it should return the NPM spec by combining the name and version`, () => {
+ const workspacePackageJSON: WorkspacesPackageJSON = {
+ ...kMinimalPackageJSON,
+ workspaces: []
+ };
+
+ const mama = new ManifestManager(workspacePackageJSON);
+
+ assert.strictEqual(
+ mama.spec,
+ `${workspacePackageJSON.name}@${workspacePackageJSON.version}`
+ );
+ });
+
+ test(`Given a WorkspacesPackageJSON with either 'name' or 'version' (or both) missing,
+ it must throw an Error`, () => {
+ const cases: WorkspacesPackageJSON[] = [
+ {
+ workspaces: ["src/a"]
+ },
+ {
+ name: "foo",
+ workspaces: ["src/a"]
+ },
+ {
+ version: "1.0.0",
+ workspaces: ["src/a"]
+ }
+ ];
+
+ for (const wsPackageJSON of cases) {
+ assert.throws(
+ () => new ManifestManager(wsPackageJSON).spec,
+ {
+ name: "Error",
+ message: "spec is not available for the given workspace"
+ }
+ )
+ }
+ });
+ });
+
+ describe("get author", () => {
+ test("Given a PackageJSON with an unparsed author field, it should parse it and return a Contact object.", () => {
+ const packageJSON: PackageJSON = {
+ ...kMinimalPackageJSON,
+ author: "John Doe "
+ };
+
+ const mama = new ManifestManager(packageJSON);
+
+ assert.deepStrictEqual(
+ mama.author,
+ {
+ name: "John Doe",
+ email: "john.doe@gmail.com"
+ }
+ );
+ });
+
+ test("Given a PackageJSON with a parsed author field, it must be returned unchanged.", () => {
+ const packageJSON: PackageJSON = {
+ ...kMinimalPackageJSON,
+ author: {
+ name: "John Doe",
+ email: "john.doe@gmail.com"
+ }
+ };
+
+ const mama = new ManifestManager(packageJSON);
+
+ assert.deepStrictEqual(
+ mama.author,
+ packageJSON.author
+ );
+ });
+
+ test("Given a PackageJSON with no author field, it must return null.", () => {
+ const packageJSON: PackageJSON = {
+ ...kMinimalPackageJSON
+ };
+
+ const mama = new ManifestManager(packageJSON);
+
+ assert.deepStrictEqual(
+ mama.author,
+ null
+ );
+ });
+ });
+
+ describe("get isWorkspace", () => {
+ test("Given a PackageJSON it must return false", () => {
+ const mama = new ManifestManager(kMinimalPackageJSON);
+
+ assert.strictEqual(
+ mama.isWorkspace,
+ false
+ );
+ });
+
+ test("Given a WorkspacesPackageJSON it must return true", () => {
+ const mama = new ManifestManager({
+ ...kMinimalPackageJSON,
+ workspaces: ["src/a"]
+ });
+
+ assert.ok(mama.isWorkspace);
+ });
+ });
+
+ describe("get integry", () => {
+ test("Given a WorkspacesPackageJSON, it must throw an error stating that workspaces are not supported.", () => {
+ const mama = new ManifestManager({
+ ...kMinimalPackageJSON,
+ workspaces: ["src/a"]
+ });
+
+ assert.throws(
+ () => mama.integrity,
+ {
+ name: "Error",
+ message: "integrity is not available for workspaces"
+ }
+ );
+ });
+
+ test("Given a minimal PackageJSON, it must return the expected hash", () => {
+ const mama = new ManifestManager(kMinimalPackageJSON);
+
+ assert.strictEqual(
+ mama.integrity,
+ kMinimalPackageJSONIntegrity
+ );
+ });
+
+ test("Given a minimal PackageJSON with a different license, it must not return the expected hash", () => {
+ const mama = new ManifestManager({
+ ...kMinimalPackageJSON,
+ license: "MIT"
+ });
+
+ assert.notStrictEqual(
+ mama.integrity,
+ kMinimalPackageJSONIntegrity
+ );
+ });
+
+ test("Given two PackageJSON files with unordered properties, the hash must be equal.", () => {
+ const mamaA = new ManifestManager({
+ ...kMinimalPackageJSON,
+ dependencies: {
+ kleur: "1.4.4"
+ },
+ scripts: {
+ start: "node index.js",
+ build: "tsc"
+ }
+ });
+ const mamaB = new ManifestManager({
+ scripts: {
+ build: "tsc",
+ start: "node index.js"
+ },
+ ...kMinimalPackageJSON,
+ dependencies: {
+ kleur: "1.4.4"
+ }
+ });
+
+ assert.strictEqual(
+ mamaA.integrity,
+ mamaB.integrity,
+ "hash values must be equal because both PackageJSON has the same set of properties"
+ );
+ });
+
+ test("Given two different PackageJSON, the hash must not be equal.", () => {
+ const mamaA = new ManifestManager({
+ ...kMinimalPackageJSON,
+ dependencies: {
+ kleur: "1.4.4"
+ }
+ });
+ const mamaB = new ManifestManager({
+ ...kMinimalPackageJSON,
+ scripts: {
+ build: "tsc",
+ start: "node index.js"
+ }
+ });
+
+ assert.notStrictEqual(
+ mamaA.integrity,
+ mamaB.integrity
+ );
+ });
+ });
+
+ describe("flags", () => {
+ test("Given a minimal PackageJSON we must verify default values", () => {
+ const mama = new ManifestManager(kMinimalPackageJSON);
+
+ assert.deepStrictEqual(
+ mama.flags,
+ {
+ hasUnsafeScripts: false,
+ isNative: false
+ }
+ );
+ });
+
+ test("Property must be Sealed", () => {
+ const mama = new ManifestManager(kMinimalPackageJSON);
+
+ assert.ok(Object.isSealed(mama.flags));
+
+ const isUpdated = Reflect.set(mama.flags, "isNative", true);
+ assert.ok(isUpdated);
+ assert.ok(mama.flags.isNative);
+ });
+
+ describe("isNative", () => {
+ it("Must equal true if either dependencies or devDependencies contains a native package", () => {
+ const cases: PackageJSON[] = [
+ {
+ ...kMinimalPackageJSON,
+ dependencies: {
+ "node-gyp": "^1.0.0"
+ }
+ },
+ {
+ ...kMinimalPackageJSON,
+ devDependencies: {
+ "node-gyp": "^1.0.0"
+ }
+ }
+ ];
+
+ for (const packageJSON of cases) {
+ const mama = new ManifestManager(packageJSON);
+ assert.ok(mama.flags.isNative);
+ }
+ });
+
+ it("Must equal true if gypfile is present and truthy", () => {
+ const packageJSON: PackageJSON = {
+ ...kMinimalPackageJSON,
+ gypfile: true
+ };
+
+ const mama = new ManifestManager(packageJSON);
+ assert.ok(mama.flags.isNative);
+ });
+
+ it(`Must equal false if gypfile is falsy and none of the dependencies or devDependencies
+ contain an identified native package to build or provide N-API features.`, () => {
+ const packageJSON: PackageJSON = {
+ ...kMinimalPackageJSON,
+ dependencies: {
+ kleur: "1.0.0"
+ },
+ devDependencies: {
+ mocha: "1.0.0"
+ },
+ gypfile: false
+ };
+
+ const mama = new ManifestManager(packageJSON);
+ assert.strictEqual(
+ mama.flags.isNative,
+ false
+ );
+ });
+ });
+
+ describe("hasUnsafeScripts", () => {
+ it("Must equal true if scripts contains at least one unsafe NPM built-in script, such as postinstall.", () => {
+ const cases: PackageJSON[] = [
+ {
+ ...kMinimalPackageJSON,
+ scripts: {
+ preinstall: ""
+ }
+ },
+ {
+ ...kMinimalPackageJSON,
+ scripts: {
+ install: ""
+ }
+ }
+ ];
+
+ for (const packageJSON of cases) {
+ const mama = new ManifestManager(packageJSON);
+ assert.ok(mama.flags.hasUnsafeScripts);
+ }
+ });
+
+ it(`Must equal false if none of the script are unsafe`, () => {
+ const packageJSON: PackageJSON = {
+ ...kMinimalPackageJSON,
+ scripts: {
+ start: "node index.js",
+ build: "tsc -b"
+ }
+ };
+
+ const mama = new ManifestManager(packageJSON);
+ assert.strictEqual(
+ mama.flags.hasUnsafeScripts,
+ false
+ );
+ });
+ });
+ });
+});
diff --git a/workspaces/mama/tsconfig.json b/workspaces/mama/tsconfig.json
new file mode 100644
index 0000000..d58c844
--- /dev/null
+++ b/workspaces/mama/tsconfig.json
@@ -0,0 +1,13 @@
+{
+ "extends": "../../tsconfig.base.json",
+ "compilerOptions": {
+ "rootDir": "src",
+ "outDir": "dist",
+ },
+ "include": ["src"],
+ "references": [
+ {
+ "path": "../npm-types"
+ }
+ ]
+}
diff --git a/workspaces/npm-types/README.md b/workspaces/npm-types/README.md
index b83c56c..4e85b5b 100644
--- a/workspaces/npm-types/README.md
+++ b/workspaces/npm-types/README.md
@@ -7,7 +7,7 @@
## Requirements
-- [Node.js](https://nodejs.org/en/) v18 or higher
+- [Node.js](https://nodejs.org/en/) v20 or higher
## Getting Started
diff --git a/workspaces/npm-types/src/index.d.ts b/workspaces/npm-types/src/index.d.ts
index 353febc..fe1acfa 100644
--- a/workspaces/npm-types/src/index.d.ts
+++ b/workspaces/npm-types/src/index.d.ts
@@ -60,8 +60,26 @@ export interface Dist {
signatures?: Signature[];
}
+export type ConditionalNodeExport = Partial>;
+
+export type NodeExport =
+ ConditionalNodeExport &
+ Record;
+
+export type NodeImport =
+ { node: string } |
+ { default: string } |
+ { node: string, default: string };
+
// this is in the tarball or the project. it really could have anything in it.
export interface PackageJSON {
+ // Required (except for workspaces)
+ name: string;
+ version: string;
+
author?: Contact | string;
bin?: Record;
browser?: Record | string;
@@ -80,9 +98,7 @@ export interface PackageJSON {
homepage?: string;
keywords?: string[];
license?: string;
- main?: string;
man?: string | string[];
- name: string;
optionalDependencies?: Record;
os?: string[];
peerDependencies?: Record;
@@ -91,12 +107,16 @@ export interface PackageJSON {
repository?: Repository | string;
scripts?: Record;
types?: string;
- version: string;
- // Node.js
+ /**
+ * @see https://nodejs.org/api/packages.html#nodejs-packagejson-field-definitions
+ * Node.js package.json field definitions
+ */
+ main?: string;
type?: "script" | "module";
- imports?: Record>;
- exports?: string | Record;
+ packageManager?: string;
+ imports?: Record<`#${string}`, string | NodeImport>;
+ exports?: string | NodeExport;
// Others
gypfile?: boolean;
@@ -104,6 +124,12 @@ export interface PackageJSON {
[field: string]: unknown;
}
+export interface WorkspacesPackageJSON extends PackageJSON {
+ name?: string;
+ version?: string;
+ workspaces: string[];
+}
+
export interface PackumentVersion extends PackageJSON {
// bugs, author, contributors, and repository can be simple strings in
// package.json, but not in registry metadata.
diff --git a/workspaces/scanner/test/depWalker.spec.ts b/workspaces/scanner/test/depWalker.spec.ts
index d840e79..be79656 100644
--- a/workspaces/scanner/test/depWalker.spec.ts
+++ b/workspaces/scanner/test/depWalker.spec.ts
@@ -63,6 +63,7 @@ test("execute depWalker on @slimio/is", async() => {
cleanupPayload(resultAsJSON);
const expectedResult = JSON.parse(readFileSync(join("test", FIXTURE_PATH, "slimio.is-result.json"), "utf-8"));
+ // console.log(JSON.stringify(resultAsJSON, null, 2));
assert.deepEqual(resultAsJSON, expectedResult);
});
diff --git a/workspaces/scanner/test/fixtures/depWalker/slimio.is-result.json b/workspaces/scanner/test/fixtures/depWalker/slimio.is-result.json
index f57bc6c..fcfc47e 100644
--- a/workspaces/scanner/test/fixtures/depWalker/slimio.is-result.json
+++ b/workspaces/scanner/test/fixtures/depWalker/slimio.is-result.json
@@ -48,7 +48,7 @@
"type": "git",
"url": "git+https://github.com/SlimIO/is.git"
},
- "integrity": "c9781c55ab750e58bed9ce2560581ff4087b8c3129462543fa6fee4e717ba2a9",
+ "integrity": "d4466c4f46f5bf6550f569f0938d0a607e0cf6e9",
"licenses": [
{
"licenses": {
diff --git a/workspaces/tarball/README.md b/workspaces/tarball/README.md
index d800137..ae6695b 100644
--- a/workspaces/tarball/README.md
+++ b/workspaces/tarball/README.md
@@ -7,7 +7,7 @@
## Requirements
-- [Node.js](https://nodejs.org/en/) v18 or higher
+- [Node.js](https://nodejs.org/en/) v20 or higher
## Getting Started
diff --git a/workspaces/tarball/package.json b/workspaces/tarball/package.json
index 4697d32..0fadf29 100644
--- a/workspaces/tarball/package.json
+++ b/workspaces/tarball/package.json
@@ -7,6 +7,7 @@
"types": "./dist/index.d.ts",
"scripts": {
"build": "tsc -b",
+ "prepublishOnly": "npm run build",
"test-only": "glob -c \"tsx --test\" \"./test/**/*.spec.ts\"",
"test": "c8 -r html npm run test-only"
},
@@ -31,14 +32,12 @@
"@nodesecure/conformance": "^1.0.0",
"@nodesecure/fs-walk": "^2.0.0",
"@nodesecure/js-x-ray": "^6.3.0",
+ "@nodesecure/mama": "^1.0.0",
"@nodesecure/npm-types": "^1.0.0",
"@nodesecure/utils": "^2.1.0",
- "builtins": "^5.1.0",
- "lodash.difference": "^4.5.0",
"pacote": "^18.0.6"
},
"devDependencies": {
- "@types/lodash.difference": "^4.5.9",
"get-folder-size": "^4.0.0"
}
}
diff --git a/workspaces/tarball/src/constants.ts b/workspaces/tarball/src/constants.ts
deleted file mode 100644
index ed3bf2f..0000000
--- a/workspaces/tarball/src/constants.ts
+++ /dev/null
@@ -1,14 +0,0 @@
-export const NPM_TOKEN = typeof process.env.NODE_SECURE_TOKEN === "string" ?
- { token: process.env.NODE_SECURE_TOKEN } :
- {};
-
-/**
- * @see https://www.nerdycode.com/prevent-npm-executing-scripts-security/
- */
-export const UNSAFE_SCRIPTS = new Set([
- "install",
- "preinstall",
- "postinstall",
- "preuninstall",
- "postuninstall"
-]);
diff --git a/workspaces/tarball/src/manifest.ts b/workspaces/tarball/src/manifest.ts
deleted file mode 100644
index 438c58a..0000000
--- a/workspaces/tarball/src/manifest.ts
+++ /dev/null
@@ -1,85 +0,0 @@
-// Import Node.js Dependencies
-import fs from "node:fs/promises";
-import path from "node:path";
-import crypto from "node:crypto";
-
-// Import Third-party Dependencies
-import type { PackageJSON } from "@nodesecure/npm-types";
-import { parseAuthor } from "@nodesecure/utils";
-
-// Import Internal Dependencies
-import { UNSAFE_SCRIPTS } from "./constants.js";
-
-// CONSTANTS
-// PR welcome to contribute to this list!
-const kNativeNpmPackages = new Set([
- "node-gyp", "node-pre-gyp", "node-gyp-build", "node-addon-api"
-]);
-const kNodemodulesBinPrefix = "node_modules/.bin/";
-
-export async function read(
- location: string
-): Promise {
- const packageStr = await fs.readFile(
- path.join(location, "package.json"),
- "utf-8"
- );
-
- return JSON.parse(packageStr);
-}
-
-export async function readAnalyze(location: string) {
- const {
- name,
- version,
- description = "",
- author = {},
- scripts = {},
- dependencies = {},
- devDependencies = {},
- gypfile = false,
- engines = {},
- repository = {},
- imports = {},
- license = ""
- } = await read(location);
-
- for (const [scriptName, scriptValue] of Object.entries(scripts)) {
- if (scriptValue.startsWith(kNodemodulesBinPrefix)) {
- scripts[scriptName] = scriptValue.replaceAll(kNodemodulesBinPrefix, "");
- }
- }
-
- const integrityObj = {
- name,
- version,
- dependencies,
- license,
- scripts
- };
-
- const integrity = crypto
- .createHash("sha256")
- .update(JSON.stringify(integrityObj))
- .digest("hex");
-
- const packageDeps = Object.keys(dependencies);
- const packageDevDeps = Object.keys(devDependencies);
- const hasNativePackage = [...packageDevDeps, ...packageDeps]
- .some((pkg) => kNativeNpmPackages.has(pkg));
-
- return {
- author: parseAuthor(author),
- description,
- engines,
- repository,
- scripts,
- hasScript: Object.keys(scripts)
- .some((value) => UNSAFE_SCRIPTS.has(value.toLowerCase())),
- packageDeps,
- packageDevDeps,
- nodejs: { imports },
- hasNativeElements: hasNativePackage || gypfile,
- integrity
- };
-}
diff --git a/workspaces/tarball/src/tarball.ts b/workspaces/tarball/src/tarball.ts
index b29d11a..e542b16 100644
--- a/workspaces/tarball/src/tarball.ts
+++ b/workspaces/tarball/src/tarball.ts
@@ -11,6 +11,7 @@ import {
} from "@nodesecure/js-x-ray";
import pacote from "pacote";
import * as conformance from "@nodesecure/conformance";
+import { ManifestManager } from "@nodesecure/mama";
// Import Internal Dependencies
import {
@@ -20,13 +21,44 @@ import {
booleanToFlags,
getSemVerWarning
} from "./utils/index.js";
-import { NPM_TOKEN } from "./constants.js";
-import * as manifest from "./manifest.js";
import * as sast from "./sast/index.js";
-import type { DependencyRef } from "./types.js";
+export interface DependencyRef {
+ id: number;
+ usedBy: Record;
+ isDevDependency: boolean;
+ existOnRemoteRegistry: boolean;
+ flags: string[];
+ description: string;
+ size: number;
+ author: Record;
+ engines: Record;
+ repository: any;
+ scripts: Record;
+ warnings: any;
+ licenses: conformance.SpdxFileLicenseConformance[];
+ uniqueLicenseIds: string[];
+ gitUrl: string | null;
+ alias: Record;
+ composition: {
+ extensions: string[];
+ files: string[];
+ minified: string[];
+ unused: string[];
+ missing: string[];
+ required_files: string[];
+ required_nodejs: string[];
+ required_thirdparty: string[];
+ required_subpath: Record;
+ }
+}
+
// CONSTANTS
+const NPM_TOKEN = typeof process.env.NODE_SECURE_TOKEN === "string" ?
+ { token: process.env.NODE_SECURE_TOKEN } :
+ {};
+
const kNativeCodeExtensions = new Set([".gyp", ".c", ".cpp", ".node", ".so", ".h"]);
const kJsExtname = new Set([".js", ".mjs", ".cjs"]);
@@ -49,35 +81,27 @@ export async function scanDirOrArchive(
// If this is an NPM tarball then we extract it on the disk with pacote.
if (isNpmTarball) {
- await pacote.extract(ref.flags.includes("isGit") ? ref.gitUrl! : `${name}@${version}`, dest, {
- ...NPM_TOKEN,
- registry,
- cache: `${os.homedir()}/.npm`
- });
- await timers.setImmediate();
- }
- else {
- // Set links to an empty object because theses are generated only for NPM tarballs
- Object.assign(ref, { links: {} });
+ await pacote.extract(
+ ref.flags.includes("isGit") ? ref.gitUrl! : `${name}@${version}`,
+ dest,
+ {
+ ...NPM_TOKEN,
+ registry,
+ cache: `${os.homedir()}/.npm`
+ }
+ );
}
// Read the package.json at the root of the directory or archive.
- const {
- packageDeps,
- packageDevDeps,
- author,
- description,
- hasScript,
- hasNativeElements,
- nodejs,
- engines,
- repository,
- scripts,
- integrity
- } = await manifest.readAnalyze(dest);
- Object.assign(ref, {
- author, description, engines, repository, scripts, integrity
- });
+ const mama = await ManifestManager.fromPackageJSON(dest);
+ {
+ const { description, engines, repository, scripts } = mama.document;
+ Object.assign(ref, {
+ description, engines, repository, scripts,
+ author: mama.author,
+ integrity: mama.integrity
+ });
+ }
// Get the composition of the (extracted) directory
const { ext, files, size } = await getTarballComposition(dest);
@@ -95,8 +119,6 @@ export async function scanDirOrArchive(
ref.size = size;
ref.composition.extensions.push(...ext);
ref.composition.files.push(...files);
- const hasBannedFile = files.some((path) => isSensitiveFile(path));
- const hasNativeCode = hasNativeElements || files.some((file) => kNativeCodeExtensions.has(path.extname(file)));
// Search for minified and runtime dependencies
// Run a JS-X-Ray analysis on each JavaScript files of the project!
@@ -125,11 +147,14 @@ export async function scanDirOrArchive(
nodeDependencies, thirdPartyDependencies, subpathImportsDependencies, missingDependencies, unusedDependencies, flags
} = analyzeDependencies(
dependencies,
- { packageDeps, packageDevDeps, tryDependencies, nodeImports: nodejs.imports }
+ {
+ mama,
+ tryDependencies
+ }
);
ref.composition.required_thirdparty = thirdPartyDependencies;
- ref.composition.required_subpath = Object.fromEntries(subpathImportsDependencies);
+ ref.composition.required_subpath = subpathImportsDependencies;
ref.composition.unused.push(...unusedDependencies);
ref.composition.missing.push(...missingDependencies);
ref.composition.required_files = filesDependencies;
@@ -137,7 +162,6 @@ export async function scanDirOrArchive(
ref.composition.minified = minifiedFiles;
// License
- await timers.setImmediate();
const { licenses, uniqueLicenseIds } = await conformance.extractLicenses(dest);
ref.licenses = licenses;
ref.uniqueLicenseIds = uniqueLicenseIds;
@@ -148,9 +172,10 @@ export async function scanDirOrArchive(
hasMultipleLicenses: uniqueLicenseIds.length > 1,
hasMinifiedCode: minifiedFiles.length > 0,
hasWarnings: ref.warnings.length > 0 && !ref.flags.includes("hasWarnings"),
- hasBannedFile,
- hasNativeCode,
- hasScript
+ hasBannedFile: files.some((path) => isSensitiveFile(path)),
+ hasNativeCode: mama.flags.isNative ||
+ files.some((file) => kNativeCodeExtensions.has(path.extname(file))),
+ hasScript: mama.flags.hasUnsafeScripts
}));
}
@@ -179,7 +204,8 @@ export async function scanPackage(
dest: string,
packageName?: string
): Promise {
- const { type = "script", name } = await manifest.read(dest);
+ const { document } = await ManifestManager.fromPackageJSON(dest);
+ const { type = "script", name } = document;
await timers.setImmediate();
const { ext, files, size } = await getTarballComposition(dest);
@@ -220,5 +246,3 @@ export async function scanPackage(
ast: { dependencies, warnings }
};
}
-
-export type { DependencyRef };
diff --git a/workspaces/tarball/src/types.ts b/workspaces/tarball/src/types.ts
deleted file mode 100644
index 4dcaa9b..0000000
--- a/workspaces/tarball/src/types.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-// Import Third-party Dependencies
-import type { SpdxFileLicenseConformance } from "@nodesecure/conformance";
-
-export interface DependencyRef {
- id: number;
- usedBy: Record;
- isDevDependency: boolean;
- existOnRemoteRegistry: boolean;
- flags: string[];
- description: string;
- size: number;
- author: Record;
- engines: Record;
- repository: any;
- scripts: Record;
- warnings: any;
- licenses: SpdxFileLicenseConformance[];
- uniqueLicenseIds: string[];
- gitUrl: string | null;
- alias: Record;
- composition: {
- extensions: string[];
- files: string[];
- minified: string[];
- unused: string[];
- missing: string[];
- required_files: string[];
- required_nodejs: string[];
- required_thirdparty: string[];
- required_subpath: Record;
- }
-}
diff --git a/workspaces/tarball/src/utils/analyzeDependencies.ts b/workspaces/tarball/src/utils/analyzeDependencies.ts
index 7f2b619..8aaad3d 100644
--- a/workspaces/tarball/src/utils/analyzeDependencies.ts
+++ b/workspaces/tarball/src/utils/analyzeDependencies.ts
@@ -1,26 +1,74 @@
+// Import Node.js Dependencies
+import path from "node:path";
+
// Import Third-party Dependencies
-import difference from "lodash.difference";
-// @ts-ignore
-import builtins from "builtins";
+import { ManifestManager } from "@nodesecure/mama";
+import type { NodeImport } from "@nodesecure/npm-types";
// Import Internal Dependencies
import { getPackageName } from "./getPackageName.js";
// CONSTANTS
-const kNodeModules = new Set(builtins({ experimental: true }));
+export const builtins = new Set([
+ "assert",
+ "buffer",
+ "child_process",
+ "cluster",
+ "console",
+ "constants",
+ "crypto",
+ "dgram",
+ "dns",
+ "domain",
+ "events",
+ "fs",
+ "http",
+ "https",
+ "module",
+ "net",
+ "os",
+ "path",
+ "punycode",
+ "querystring",
+ "readline",
+ "repl",
+ "stream",
+ "string_decoder",
+ "sys",
+ "timers",
+ "tls",
+ "tty",
+ "url",
+ "util",
+ "vm",
+ "zlib",
+ "freelist",
+ "v8",
+ "process",
+ "inspector",
+ "async_hooks",
+ "http2",
+ "perf_hooks",
+ "trace_events",
+ "worker_threads",
+ "node:test",
+ "wasi",
+ "diagnostics_channel"
+]);
+
const kExternalModules = new Set(["http", "https", "net", "http2", "dgram", "child_process"]);
export interface analyzeDependenciesOptions {
- packageDeps: string[];
- packageDevDeps: string[];
+ mama:
+ Pick &
+ Partial>;
tryDependencies: Set;
- nodeImports?: Record;
}
export interface analyzeDependenciesResult {
nodeDependencies: string[];
thirdPartyDependencies: string[];
- subpathImportsDependencies: [string, string][];
+ subpathImportsDependencies: Record;
unusedDependencies: string[];
missingDependencies: string[];
flags: {
@@ -30,33 +78,41 @@ export interface analyzeDependenciesResult {
}
export function analyzeDependencies(
- dependencies: string[],
+ sourceDependencies: string[],
options: analyzeDependenciesOptions
): analyzeDependenciesResult {
- const { packageDeps, packageDevDeps, tryDependencies, nodeImports = {} } = options;
+ const { mama, tryDependencies } = options;
+ const { dependencies, devDependencies, nodejsImports = {} } = mama;
// See: https://nodejs.org/api/packages.html#subpath-imports
- const subpathImportsDependencies = dependencies
- .filter((name) => isAliasDependency(name) && name in nodeImports)
- .map((name) => buildSubpathDependency(name, nodeImports));
+ const subpathImportsDependencies = Object.fromEntries(
+ sourceDependencies
+ .filter((name) => isAliasFileModule(name) && name in nodejsImports)
+ .map((name) => buildSubpathDependency(name, nodejsImports))
+ );
const thirdPartyDependenciesAliased = new Set(
- subpathImportsDependencies.flat().filter((name) => !isAliasDependency(name))
+ Object.values(subpathImportsDependencies).filter((mod) => !isFile(mod))
);
- const thirdPartyDependencies = dependencies
- .map((name) => (packageDeps.includes(name) ? name : getPackageName(name)))
- .filter((name) => !name.startsWith("."))
- .filter((name) => !isNodeCoreModule(name))
- .filter((name) => !packageDevDeps.includes(name))
- .filter((name) => !tryDependencies.has(name));
+ const thirdPartyDependencies = sourceDependencies.flatMap((sourceName) => {
+ const name = dependencies.includes(sourceName) ? sourceName : getPackageName(sourceName);
+
+ return isFile(name) ||
+ isCoreModule(name) ||
+ devDependencies.includes(name) ||
+ tryDependencies.has(name) ?
+ [] : name;
+ });
const unusedDependencies = difference(
- packageDeps.filter((name) => !name.startsWith("@types")),
+ dependencies.filter((name) => !name.startsWith("@types")),
[...thirdPartyDependencies, ...thirdPartyDependenciesAliased]
);
- const missingDependencies = [...new Set(difference(thirdPartyDependencies, packageDeps))]
- .filter((name: string) => !(name in nodeImports));
- const nodeDependencies = dependencies.filter((name) => isNodeCoreModule(name));
+ const missingDependencies = [
+ ...new Set(difference(thirdPartyDependencies, dependencies))
+ ]
+ .filter((name: string) => !(name in nodejsImports) && !thirdPartyDependenciesAliased.has(name));
+ const nodeDependencies = sourceDependencies.filter((name) => isCoreModule(name));
return {
nodeDependencies,
@@ -72,27 +128,38 @@ export function analyzeDependencies(
};
}
-/**
- * @param {!string} moduleName
- * @returns {boolean}
- */
-function isNodeCoreModule(moduleName: string): boolean {
+function difference(arr1: T[], arr2: T[]): T[] {
+ return arr1.filter(item => !arr2.includes(item));
+}
+
+function isFile(
+ name: string
+) {
+ return name.startsWith(".") || path.extname(name) !== ""
+}
+
+function isCoreModule(
+ moduleName: string
+): boolean {
const cleanModuleName = moduleName.startsWith("node:") ? moduleName.slice(5) : moduleName;
// Note: We need to also check moduleName because builtins package only return true for 'node:test'.
- return kNodeModules.has(cleanModuleName) || kNodeModules.has(moduleName);
+ return builtins.has(cleanModuleName) || builtins.has(moduleName);
}
-function isAliasDependency(moduleName: string): boolean {
+function isAliasFileModule(
+ moduleName: string
+): moduleName is `#${string}` {
return moduleName.charAt(0) === "#";
}
function buildSubpathDependency(
alias: string,
- nodeImports: Record
+ nodeImports: Record
): [string, string] {
const importEntry = nodeImports[alias]!;
- const importedDependency = importEntry.node ?? importEntry.default;
- return [alias, importedDependency];
+ return typeof importEntry === "string" ?
+ [alias, importEntry] :
+ [alias, "node" in importEntry ? importEntry.node : importEntry.default];
}
diff --git a/workspaces/tarball/test/manifest.spec.ts b/workspaces/tarball/test/manifest.spec.ts
deleted file mode 100644
index 15473d4..0000000
--- a/workspaces/tarball/test/manifest.spec.ts
+++ /dev/null
@@ -1,137 +0,0 @@
-// Import Node.js Dependencies
-import fs from "node:fs/promises";
-import path from "node:path";
-import crypto from "node:crypto";
-import assert from "node:assert";
-import { test } from "node:test";
-
-// Import Third-party Dependencies
-import sinon from "sinon";
-
-// Import Internal Dependencies
-import * as manifest from "../src/manifest.js";
-
-test("manifest.readAnalyze with a fake empty package.json (so all default values must be returned)", async() => {
- const readFile = sinon.stub(fs, "readFile").resolves(JSON.stringify({}));
-
- try {
- const manifestResult = await manifest.readAnalyze(process.cwd());
-
- assert.deepEqual(manifestResult.packageDeps, []);
- assert.deepEqual(manifestResult.packageDevDeps, []);
- assert.ok(!manifestResult.hasNativeElements);
- assert.ok(!manifestResult.hasScript);
- assert.deepEqual(manifestResult.author, null);
- assert.strictEqual(manifestResult.description, "");
- assert.deepEqual(manifestResult.nodejs, { imports: {} });
- assert.ok(readFile.calledWith(path.join(process.cwd(), "package.json"), "utf-8"));
- assert.ok(readFile.calledOnce);
- }
- finally {
- readFile.restore();
- }
-});
-
-test("manifest.readAnalyze with a fake but consistent data", async() => {
- const readFile = sinon.stub(fs, "readFile").resolves(JSON.stringify({
- description: "foobar",
- author: "GENTILHOMME Thomas ",
- scripts: {
- preinstall: "npx foobar"
- },
- dependencies: {
- "@slimio/is": "^1.0.0"
- },
- devDependencies: {
- mocha: ">=2.5.0"
- },
- imports: {
- "#dep": {
- node: "kleur"
- }
- },
- gypfile: true
- }));
-
- try {
- const manifestResult = await manifest.readAnalyze(process.cwd());
-
- assert.deepEqual(manifestResult.packageDeps, ["@slimio/is"]);
- assert.deepEqual(manifestResult.packageDevDeps, ["mocha"]);
- assert.deepEqual(manifestResult.nodejs.imports, {
- "#dep": {
- node: "kleur"
- }
- });
- assert.deepEqual(manifestResult.scripts, {
- preinstall: "npx foobar"
- });
- assert.deepEqual(manifestResult.engines, {});
- assert.deepEqual(manifestResult.repository, {});
- assert.ok(manifestResult.hasNativeElements);
- assert.ok(manifestResult.hasScript);
- assert.deepEqual(manifestResult.author, {
- name: "GENTILHOMME Thomas",
- email: "gentilhomme.thomas@gmail.com"
- });
- assert.strictEqual(manifestResult.description, "foobar");
-
- assert.ok(readFile.calledWith(path.join(process.cwd(), "package.json"), "utf-8"));
- assert.ok(readFile.calledOnce);
- }
- finally {
- readFile.restore();
- }
-});
-
-test("manifest.readAnalyze should return hasNativeElements: true because of the dependencies", async() => {
- const readFile = sinon.stub(fs, "readFile").resolves(JSON.stringify({
- dependencies: {
- "node-addon-api": "^1.0.0"
- },
- devDependencies: {
- "node-gyp": "8.0.0"
- }
- }));
-
- try {
- const manifestResult = await manifest.readAnalyze(process.cwd());
- assert.ok(manifestResult.hasNativeElements);
- }
- finally {
- readFile.restore();
- }
-});
-
-test("manifest.readAnalyze should generate the proper integrity", async() => {
- // Warning: property must be in the same order
- const manifestJSON = {
- name: "foo",
- version: "1.0.0",
- dependencies: {
- "node-addon-api": "^1.0.0"
- },
- license: "MIT",
- scripts: {
- test: "hello world!"
- }
- };
- const expectedIntegrity = crypto
- .createHash("sha256")
- .update(JSON.stringify(manifestJSON))
- .digest("hex");
-
- const readFile = sinon.stub(fs, "readFile").resolves(JSON.stringify(manifestJSON));
-
- try {
- const manifestResult = await manifest.readAnalyze(process.cwd());
-
- assert.equal(
- manifestResult.integrity,
- expectedIntegrity
- );
- }
- finally {
- readFile.restore();
- }
-});
diff --git a/workspaces/tarball/test/utils/analyzeDependencies.spec.ts b/workspaces/tarball/test/utils/analyzeDependencies.spec.ts
index 359db01..79b486d 100644
--- a/workspaces/tarball/test/utils/analyzeDependencies.spec.ts
+++ b/workspaces/tarball/test/utils/analyzeDependencies.spec.ts
@@ -6,18 +6,20 @@ import assert from "node:assert";
import { analyzeDependencies } from "../../src/utils/index.js";
test("analyzeDependencies should detect Node.js dependencies and also flag hasExternalCapacity", () => {
- const packageDeps = [];
- const packageDevDeps = [];
+ const mama = {
+ dependencies: [],
+ devDependencies: []
+ };
const result = analyzeDependencies([
"fs",
"http"
- ], { packageDeps, packageDevDeps, tryDependencies: new Set() });
+ ], { mama, tryDependencies: new Set() });
assert.deepEqual(result, {
nodeDependencies: ["fs", "http"],
thirdPartyDependencies: [],
- subpathImportsDependencies: [],
+ subpathImportsDependencies: {},
unusedDependencies: [],
missingDependencies: [],
flags: { hasExternalCapacity: true, hasMissingOrUnusedDependency: false }
@@ -25,19 +27,21 @@ test("analyzeDependencies should detect Node.js dependencies and also flag hasEx
});
test("analyzeDependencies should detect prefixed (namespaced 'node:') core dependencies", () => {
- const packageDeps = ["node:foo"];
- const packageDevDeps = [];
+ const mama = {
+ dependencies: ["node:foo"],
+ devDependencies: []
+ };
const result = analyzeDependencies([
"node:fs",
"node:foo",
"node:bar"
- ], { packageDeps, packageDevDeps, tryDependencies: new Set() });
+ ], { mama, tryDependencies: new Set() });
assert.deepEqual(result, {
nodeDependencies: ["node:fs"],
thirdPartyDependencies: ["node:foo", "node:bar"],
- subpathImportsDependencies: [],
+ subpathImportsDependencies: {},
unusedDependencies: [],
missingDependencies: ["node:bar"],
flags: { hasExternalCapacity: false, hasMissingOrUnusedDependency: true }
@@ -45,18 +49,20 @@ test("analyzeDependencies should detect prefixed (namespaced 'node:') core depen
});
test("analyzeDependencies should be capable of detecting unused dependency 'koa'", () => {
- const packageDeps = ["koa", "kleur"];
- const packageDevDeps = ["mocha"];
+ const mama = {
+ dependencies: ["koa", "kleur"],
+ devDependencies: ["mocha"]
+ };
const result = analyzeDependencies([
"mocha",
"kleur"
- ], { packageDeps, packageDevDeps, tryDependencies: new Set() });
+ ], { mama, tryDependencies: new Set() });
assert.deepEqual(result, {
nodeDependencies: [],
thirdPartyDependencies: ["kleur"],
- subpathImportsDependencies: [],
+ subpathImportsDependencies: {},
unusedDependencies: ["koa"],
missingDependencies: [],
flags: { hasExternalCapacity: false, hasMissingOrUnusedDependency: true }
@@ -64,18 +70,20 @@ test("analyzeDependencies should be capable of detecting unused dependency 'koa'
});
test("analyzeDependencies should be capable of detecting unused dependency 'kleur'", () => {
- const packageDeps = ["mocha"];
- const packageDevDeps = [];
+ const mama = {
+ dependencies: ["mocha"],
+ devDependencies: []
+ };
const result = analyzeDependencies([
"mocha",
"kleur"
- ], { packageDeps, packageDevDeps, tryDependencies: new Set() });
+ ], { mama, tryDependencies: new Set() });
assert.deepEqual(result, {
nodeDependencies: [],
thirdPartyDependencies: ["mocha", "kleur"],
- subpathImportsDependencies: [],
+ subpathImportsDependencies: {},
unusedDependencies: [],
missingDependencies: ["kleur"],
flags: { hasExternalCapacity: false, hasMissingOrUnusedDependency: true }
@@ -83,17 +91,19 @@ test("analyzeDependencies should be capable of detecting unused dependency 'kleu
});
test("analyzeDependencies should ignore '@types' for third-party dependencies", () => {
- const packageDeps = ["@types/npm"];
- const packageDevDeps = ["kleur"];
+ const mama = {
+ dependencies: ["@types/npm"],
+ devDependencies: ["kleur"]
+ };
const result = analyzeDependencies([
"kleur"
- ], { packageDeps, packageDevDeps, tryDependencies: new Set() });
+ ], { mama, tryDependencies: new Set() });
assert.deepEqual(result, {
nodeDependencies: [],
thirdPartyDependencies: [],
- subpathImportsDependencies: [],
+ subpathImportsDependencies: {},
unusedDependencies: [],
missingDependencies: [],
flags: { hasExternalCapacity: false, hasMissingOrUnusedDependency: false }
@@ -101,19 +111,21 @@ test("analyzeDependencies should ignore '@types' for third-party dependencies",
});
test("analyzeDependencies should ignore file dependencies and try dependencies", () => {
- const packageDeps = [];
- const packageDevDeps = ["kleur"];
+ const mama = {
+ dependencies: [],
+ devDependencies: ["kleur"]
+ };
const result = analyzeDependencies([
"kleur",
"httpie",
"./foobar.js"
- ], { packageDeps, packageDevDeps, tryDependencies: new Set(["httpie"]) });
+ ], { mama, tryDependencies: new Set(["httpie"]) });
assert.deepEqual(result, {
nodeDependencies: [],
thirdPartyDependencies: [],
- subpathImportsDependencies: [],
+ subpathImportsDependencies: {},
unusedDependencies: [],
missingDependencies: [],
flags: { hasExternalCapacity: false, hasMissingOrUnusedDependency: false }
@@ -121,51 +133,61 @@ test("analyzeDependencies should ignore file dependencies and try dependencies",
});
test("analyzeDependencies should detect Node.js subpath import and set relation between #dep and kleur.", () => {
- const packageDeps = ["kleur"];
- const packageDevDeps = [];
- const nodeImports = {
- "#dep": {
- node: "kleur"
+ const mama = {
+ dependencies: ["kleur"],
+ devDependencies: [],
+ nodejsImports: {
+ "#dep": {
+ node: "kleur"
+ }
}
};
const result = analyzeDependencies([
"#dep"
- ], { packageDeps, packageDevDeps, tryDependencies: new Set(), nodeImports });
+ ], { mama, tryDependencies: new Set() });
assert.deepEqual(result, {
nodeDependencies: [],
thirdPartyDependencies: ["#dep"],
- subpathImportsDependencies: [
- ["#dep", "kleur"]
- ],
+ subpathImportsDependencies: {
+ "#dep": "kleur"
+ },
unusedDependencies: [],
missingDependencies: [],
- flags: { hasExternalCapacity: false, hasMissingOrUnusedDependency: false }
+ flags: {
+ hasExternalCapacity: false,
+ hasMissingOrUnusedDependency: false
+ }
});
});
test("analyzeDependencies should detect Node.js subpath import (with a default property pointing to a file)", () => {
- const packageDeps = ["kleur"];
- const packageDevDeps = [];
- const nodeImports = {
- "#dep": {
- default: "./foo.js"
+ const mama = {
+ dependencies: ["kleur"],
+ devDependencies: [],
+ nodejsImports: {
+ "#dep": {
+ default: "./foo.js"
+ }
}
};
const result = analyzeDependencies([
"#dep"
- ], { packageDeps, packageDevDeps, tryDependencies: new Set(), nodeImports });
+ ], { mama, tryDependencies: new Set() });
assert.deepEqual(result, {
nodeDependencies: [],
thirdPartyDependencies: ["#dep"],
- subpathImportsDependencies: [
- ["#dep", "./foo.js"]
- ],
+ subpathImportsDependencies: {
+ "#dep": "./foo.js"
+ },
unusedDependencies: ["kleur"],
missingDependencies: [],
- flags: { hasExternalCapacity: false, hasMissingOrUnusedDependency: true }
+ flags: {
+ hasExternalCapacity: false,
+ hasMissingOrUnusedDependency: true
+ }
});
});
diff --git a/workspaces/tarball/tsconfig.json b/workspaces/tarball/tsconfig.json
index e06e875..c7e53d6 100644
--- a/workspaces/tarball/tsconfig.json
+++ b/workspaces/tarball/tsconfig.json
@@ -11,6 +11,9 @@
},
{
"path": "../conformance"
+ },
+ {
+ "path": "../mama"
}
]
}
diff --git a/workspaces/tree-walker/README.md b/workspaces/tree-walker/README.md
index c4fcbd5..a14f347 100644
--- a/workspaces/tree-walker/README.md
+++ b/workspaces/tree-walker/README.md
@@ -7,7 +7,7 @@
## Requirements
-- [Node.js](https://nodejs.org/en/) v18 or higher
+- [Node.js](https://nodejs.org/en/) v20 or higher
## Getting Started
From 78b47417a5718af6b3d18be805ce3986788ab852 Mon Sep 17 00:00:00 2001
From: fraxken
Date: Sun, 7 Jul 2024 21:36:01 +0200
Subject: [PATCH 2/2] refactor(scanDirArchive): further cleanup of the
implementation
---
workspaces/scanner/test/verify.spec.ts | 2 +-
workspaces/tarball/src/sast/file.ts | 23 ++-
workspaces/tarball/src/tarball.ts | 144 +++++++++---------
workspaces/tarball/src/utils/index.ts | 2 -
.../getSemverWarning.ts => warnings.ts} | 13 ++
.../tarball/test/tarball/scanPackage.spec.ts | 1 +
...SemverWarning.spec.ts => warnings.spec.ts} | 2 +-
7 files changed, 107 insertions(+), 80 deletions(-)
rename workspaces/tarball/src/{utils/getSemverWarning.ts => warnings.ts} (56%)
rename workspaces/tarball/test/{utils/getSemverWarning.spec.ts => warnings.spec.ts} (92%)
diff --git a/workspaces/scanner/test/verify.spec.ts b/workspaces/scanner/test/verify.spec.ts
index ab65cee..9b87276 100644
--- a/workspaces/scanner/test/verify.spec.ts
+++ b/workspaces/scanner/test/verify.spec.ts
@@ -35,7 +35,7 @@ test("verify 'express' package", async() => {
"lib\\view.js",
"package.json"
].map((location) => location.replaceAll("\\", path.sep)),
- extensions: [".md", ".js", ".json"].sort(),
+ extensions: ["", ".md", ".js", ".json"].sort(),
minified: []
});
assert.ok(data.directorySize > 0);
diff --git a/workspaces/tarball/src/sast/file.ts b/workspaces/tarball/src/sast/file.ts
index f6b07cf..d8a953b 100644
--- a/workspaces/tarball/src/sast/file.ts
+++ b/workspaces/tarball/src/sast/file.ts
@@ -13,6 +13,9 @@ import {
filterDependencyKind
} from "../utils/index.js";
+// CONSTANTS
+const kJsExtname = new Set([".js", ".mjs", ".cjs"]);
+
export interface scanFileReport {
file: string;
warnings: (Omit, "value"> & { file: string; })[];
@@ -23,12 +26,12 @@ export interface scanFileReport {
}
export async function scanFile(
- dest: string,
+ destination: string,
file: string,
packageName: string
): Promise {
const result = await runASTAnalysisOnFile(
- path.join(dest, file),
+ path.join(destination, file),
{ packageName }
);
@@ -58,3 +61,19 @@ export async function scanFile(
filesDependencies: []
};
}
+
+export async function scanManyFiles(
+ files: string[],
+ destination: string,
+ packageName: string
+): Promise {
+ const scannedFiles = await Promise.allSettled(
+ files
+ .filter((fileName) => kJsExtname.has(path.extname(fileName)))
+ .map((file) => scanFile(destination, file, packageName))
+ );
+
+ return scannedFiles
+ .filter((result) => result.status === "fulfilled")
+ .map((result) => result.value);
+}
diff --git a/workspaces/tarball/src/tarball.ts b/workspaces/tarball/src/tarball.ts
index e542b16..3d64d75 100644
--- a/workspaces/tarball/src/tarball.ts
+++ b/workspaces/tarball/src/tarball.ts
@@ -18,9 +18,9 @@ import {
getTarballComposition,
isSensitiveFile,
analyzeDependencies,
- booleanToFlags,
- getSemVerWarning
+ booleanToFlags
} from "./utils/index.js";
+import * as warnings from "./warnings.js";
import * as sast from "./sast/index.js";
export interface DependencyRef {
@@ -93,7 +93,16 @@ export async function scanDirOrArchive(
}
// Read the package.json at the root of the directory or archive.
- const mama = await ManifestManager.fromPackageJSON(dest);
+ const [
+ mama,
+ composition,
+ spdx
+ ] = await Promise.all([
+ ManifestManager.fromPackageJSON(dest),
+ getTarballComposition(dest),
+ conformance.extractLicenses(dest)
+ ]);
+
{
const { description, engines, repository, scripts } = mama.document;
Object.assign(ref, {
@@ -102,57 +111,43 @@ export async function scanDirOrArchive(
integrity: mama.integrity
});
}
+ ref.licenses = spdx.licenses;
+ ref.uniqueLicenseIds = spdx.uniqueLicenseIds;
// Get the composition of the (extracted) directory
- const { ext, files, size } = await getTarballComposition(dest);
- if (files.length === 1 && files.includes("package.json")) {
- ref.warnings.push({
- kind: "empty-package",
- location: null,
- i18n: "sast_warnings.emptyPackage",
- severity: "Critical",
- source: "Scanner",
- experimental: false
- });
+ if (composition.files.length === 1 && composition.files.includes("package.json")) {
+ ref.warnings.push(warnings.getEmptyPackageWarning());
}
- ref.size = size;
- ref.composition.extensions.push(...ext);
- ref.composition.files.push(...files);
-
// Search for minified and runtime dependencies
// Run a JS-X-Ray analysis on each JavaScript files of the project!
- const fileAnalysisRaw = await Promise.allSettled(
- files
- .filter((name) => kJsExtname.has(path.extname(name)))
- .map((file) => sast.scanFile(dest, file, name))
- );
-
- const fileAnalysisResults = fileAnalysisRaw
- .filter((promiseSettledResult) => promiseSettledResult.status === "fulfilled")
- .map((promiseSettledResult) => (promiseSettledResult as PromiseFulfilledResult).value);
-
- ref.warnings.push(...fileAnalysisResults.flatMap((row) => row.warnings));
+ const scannedFiles = await sast.scanManyFiles(composition.files, dest, name);
+ ref.warnings.push(...scannedFiles.flatMap((row) => row.warnings));
if (/^0(\.\d+)*$/.test(version)) {
- ref.warnings.push(getSemVerWarning(version));
+ ref.warnings.push(warnings.getSemVerWarning(version));
}
- const dependencies = [...new Set(fileAnalysisResults.flatMap((row) => row.dependencies))];
- const filesDependencies = [...new Set(fileAnalysisResults.flatMap((row) => row.filesDependencies))];
- const tryDependencies = new Set(fileAnalysisResults.flatMap((row) => row.tryDependencies));
- const minifiedFiles = fileAnalysisResults.filter((row) => row.isMinified).flatMap((row) => row.file);
+ const dependencies = [...new Set(scannedFiles.flatMap((row) => row.dependencies))];
+ const filesDependencies = [...new Set(scannedFiles.flatMap((row) => row.filesDependencies))];
+ const tryDependencies = new Set(scannedFiles.flatMap((row) => row.tryDependencies));
+ const minifiedFiles = scannedFiles.filter((row) => row.isMinified).flatMap((row) => row.file);
const {
- nodeDependencies, thirdPartyDependencies, subpathImportsDependencies, missingDependencies, unusedDependencies, flags
+ nodeDependencies,
+ thirdPartyDependencies,
+ subpathImportsDependencies,
+ missingDependencies,
+ unusedDependencies,
+ flags
} = analyzeDependencies(
dependencies,
- {
- mama,
- tryDependencies
- }
+ { mama, tryDependencies }
);
+ ref.size = composition.size;
+ ref.composition.extensions.push(...composition.ext);
+ ref.composition.files.push(...composition.files);
ref.composition.required_thirdparty = thirdPartyDependencies;
ref.composition.required_subpath = subpathImportsDependencies;
ref.composition.unused.push(...unusedDependencies);
@@ -161,20 +156,15 @@ export async function scanDirOrArchive(
ref.composition.required_nodejs = nodeDependencies;
ref.composition.minified = minifiedFiles;
- // License
- const { licenses, uniqueLicenseIds } = await conformance.extractLicenses(dest);
- ref.licenses = licenses;
- ref.uniqueLicenseIds = uniqueLicenseIds;
-
ref.flags.push(...booleanToFlags({
...flags,
- hasNoLicense: uniqueLicenseIds.length === 0,
- hasMultipleLicenses: uniqueLicenseIds.length > 1,
+ hasNoLicense: spdx.uniqueLicenseIds.length === 0,
+ hasMultipleLicenses: spdx.uniqueLicenseIds.length > 1,
hasMinifiedCode: minifiedFiles.length > 0,
hasWarnings: ref.warnings.length > 0 && !ref.flags.includes("hasWarnings"),
- hasBannedFile: files.some((path) => isSensitiveFile(path)),
+ hasBannedFile: composition.files.some((path) => isSensitiveFile(path)),
hasNativeCode: mama.flags.isNative ||
- files.some((file) => kNativeCodeExtensions.has(path.extname(file))),
+ composition.files.some((file) => kNativeCodeExtensions.has(path.extname(file))),
hasScript: mama.flags.hasUnsafeScripts
}));
}
@@ -204,45 +194,51 @@ export async function scanPackage(
dest: string,
packageName?: string
): Promise {
- const { document } = await ManifestManager.fromPackageJSON(dest);
- const { type = "script", name } = document;
-
- await timers.setImmediate();
- const { ext, files, size } = await getTarballComposition(dest);
- ext.delete("");
+ const [
+ mama,
+ composition,
+ spdx
+ ] = await Promise.all([
+ ManifestManager.fromPackageJSON(dest),
+ getTarballComposition(dest),
+ conformance.extractLicenses(dest)
+ ])
+ const { type = "script" } = mama.document;
// Search for runtime dependencies
const dependencies: Record> = Object.create(null);
const minified: string[] = [];
const warnings: Warning[] = [];
- const JSFiles = files.filter((name) => kJsExtname.has(path.extname(name)));
+ const JSFiles = composition.files
+ .filter((name) => kJsExtname.has(path.extname(name)));
for (const file of JSFiles) {
- const result = await runASTAnalysisOnFile(path.join(dest, file), {
- packageName: packageName ?? name,
- module: type === "module"
- });
+ const result = await runASTAnalysisOnFile(
+ path.join(dest, file),
+ {
+ packageName: packageName ?? mama.document.name,
+ module: type === "module"
+ }
+ );
- warnings.push(...result.warnings.map((curr) => Object.assign({}, curr, { file })));
- if (!result.ok) {
- continue;
+ warnings.push(
+ ...result.warnings.map((curr) => Object.assign({}, curr, { file }))
+ );
+ if (result.ok) {
+ dependencies[file] = result.dependencies.dependencies;
+ result.isMinified && minified.push(file);
}
-
- dependencies[file] = result.dependencies.dependencies;
- result.isMinified && minified.push(file);
}
- await timers.setImmediate();
- const {
- uniqueLicenseIds,
- licenses
- } = await conformance.extractLicenses(dest);
-
return {
- files: { list: files, extensions: [...ext], minified },
- directorySize: size,
- uniqueLicenseIds,
- licenses,
+ files: {
+ list: composition.files,
+ extensions: [...composition.ext],
+ minified
+ },
+ directorySize: composition.size,
+ uniqueLicenseIds: spdx.uniqueLicenseIds,
+ licenses: spdx.licenses,
ast: { dependencies, warnings }
};
}
diff --git a/workspaces/tarball/src/utils/index.ts b/workspaces/tarball/src/utils/index.ts
index fd5f684..9a308ea 100644
--- a/workspaces/tarball/src/utils/index.ts
+++ b/workspaces/tarball/src/utils/index.ts
@@ -4,5 +4,3 @@ export * from "./isSensitiveFile.js";
export * from "./getPackageName.js";
export * from "./getTarballComposition.js";
export * from "./filterDependencyKind.js";
-export * from "./getSemverWarning.js";
-
diff --git a/workspaces/tarball/src/utils/getSemverWarning.ts b/workspaces/tarball/src/warnings.ts
similarity index 56%
rename from workspaces/tarball/src/utils/getSemverWarning.ts
rename to workspaces/tarball/src/warnings.ts
index 1012e54..c537855 100644
--- a/workspaces/tarball/src/utils/getSemverWarning.ts
+++ b/workspaces/tarball/src/warnings.ts
@@ -15,3 +15,16 @@ export function getSemVerWarning(
experimental: false
};
}
+
+export function getEmptyPackageWarning(): WarningDefault<"empty-package"> {
+ return {
+ kind: "empty-package",
+ file: "package.json",
+ value: "package.json",
+ location: null,
+ i18n: "sast_warnings.emptyPackage",
+ severity: "Critical",
+ source: "Scanner",
+ experimental: false
+ }
+}
diff --git a/workspaces/tarball/test/tarball/scanPackage.spec.ts b/workspaces/tarball/test/tarball/scanPackage.spec.ts
index 4a69914..37b4558 100644
--- a/workspaces/tarball/test/tarball/scanPackage.spec.ts
+++ b/workspaces/tarball/test/tarball/scanPackage.spec.ts
@@ -27,6 +27,7 @@ test("scanPackage (caseone)", async() => {
"src\\other.min.js"
].map((location) => location.replace(/\\/g, path.sep)),
extensions: [
+ "",
".txt",
".js",
".json"
diff --git a/workspaces/tarball/test/utils/getSemverWarning.spec.ts b/workspaces/tarball/test/warnings.spec.ts
similarity index 92%
rename from workspaces/tarball/test/utils/getSemverWarning.spec.ts
rename to workspaces/tarball/test/warnings.spec.ts
index 0f61990..90b5f40 100644
--- a/workspaces/tarball/test/utils/getSemverWarning.spec.ts
+++ b/workspaces/tarball/test/warnings.spec.ts
@@ -3,7 +3,7 @@ import assert from "node:assert";
import { test } from "node:test";
// Import Internal Dependencies
-import { getSemVerWarning } from "../../src/utils/index.js";
+import { getSemVerWarning } from "../src/warnings.js";
// CONSTANTS
const kDefaultWarning = {