diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2b56031..5a277e6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - node-version: [23.x, 22.x, 20.x, 18.x, 16.x, 14.x, 12.x] + node-version: [23.x, 22.x, 21.x, 20.x] steps: - uses: actions/checkout@v2 diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..327d902 --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,9 @@ +{ + "singleQuote": true, + "useTabs": true, + "tabWidth": 4, + "semi": true, + "trailingComma": "es5", + "printWidth": 140, + "endOfLine": "crlf" +} diff --git a/bin/index.js b/bin/index.js index e8ee9bf..7c3d154 100644 --- a/bin/index.js +++ b/bin/index.js @@ -1,2 +1,2 @@ #!/usr/bin/env node -require('../dist/cli.js'); +import '../dist/cli.js'; diff --git a/build/download-files.ts b/build/download-files.ts index 5676638..e155758 100644 --- a/build/download-files.ts +++ b/build/download-files.ts @@ -1,10 +1,9 @@ #!/usr/bin/env tsx import FS from 'fs'; -import Path from 'path'; import YAML from 'js-yaml'; - -import loadFile, { parseGeneratedDataFile } from '../src/helpers/load-data'; +import Path from 'path'; +import loadFile, { parseGeneratedDataFile } from '../src/program/data/loadData.ts'; async function writeFile(filename: string) { const filePath = Path.resolve('ext', filename); diff --git a/package-lock.json b/package-lock.json index a4a2d46..98d6bb4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,20 +1,19 @@ { "name": "linguist-js", - "version": "2.9.0", - "lockfileVersion": 2, + "version": "3.0.0-dev", + "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "linguist-js", - "version": "2.9.0", + "version": "3.0.0-dev", "license": "ISC", "dependencies": { - "binary-extensions": "^2.3.0 <3", - "commander": "^9.5.0 <10", + "binary-extensions": "^3.1.0", + "commander": "^14.0.0", "common-path-prefix": "^3.0.0", - "cross-fetch": "^3.2.0 <4", - "ignore": "^7.0.3", - "isbinaryfile": "^4.0.10 <5", + "ignore": "^7.0.5", + "isbinaryfile": "^5.0.6", "js-yaml": "^4.1.0", "node-cache": "^5.1.2" }, @@ -24,13 +23,14 @@ }, "devDependencies": { "@types/js-yaml": "^4.0.9", - "@types/node": "ts5.0", + "@types/node": "^24.3.1", "deep-object-diff": "^1.1.9", - "typescript": "~5.0.4 <5.1" + "prettier": "^3.6.2", + "typescript": "^5.9.2" }, "engines": { - "node": ">=12", - "npm": "<9" + "node": ">=24", + "npm": ">=10" } }, "node_modules/@types/js-yaml": { @@ -40,12 +40,12 @@ "dev": true }, "node_modules/@types/node": { - "version": "22.13.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.1.tgz", - "integrity": "sha512-jK8uzQlrvXqEU91UxiK5J7pKHyzgnI1Qnl0QDHIgVGuolJhRb9EEl28Cj9b3rGR8B2lhFCtvIm5os8lFnO/1Ew==", + "version": "24.3.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.3.1.tgz", + "integrity": "sha512-3vXmQDXy+woz+gnrTvuvNrPzekOi+Ds0ReMxw0LzBiK3a+1k0kQn9f2NWk+lgD4rJehFUmYy2gMhJ2ZI+7YP9g==", "dev": true, "dependencies": { - "undici-types": "~6.20.0" + "undici-types": "~7.10.0" } }, "node_modules/argparse": { @@ -54,11 +54,11 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, "node_modules/binary-extensions": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", - "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-3.1.0.tgz", + "integrity": "sha512-Jvvd9hy1w+xUad8+ckQsWA/V1AoyubOvqn0aygjMOVM4BfIaRav1NFS3LsTSDaV4n4FtcCtQXvzep1E6MboqwQ==", "engines": { - "node": ">=8" + "node": ">=18.20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -73,11 +73,11 @@ } }, "node_modules/commander": { - "version": "9.5.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", - "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.0.tgz", + "integrity": "sha512-2uM9rYjPvyq39NwLRqaiLtWHyDC1FvryJDa2ATTVims5YAS4PupsEQsDvP14FqhFr0P49CYDugi59xaxJlTXRA==", "engines": { - "node": "^12.20.0 || >=14" + "node": ">=20" } }, "node_modules/common-path-prefix": { @@ -85,14 +85,6 @@ "resolved": "https://registry.npmjs.org/common-path-prefix/-/common-path-prefix-3.0.0.tgz", "integrity": "sha512-QE33hToZseCH3jS0qN96O/bSh3kaw/h+Tq7ngyY9eWDUnTlTNUyqfqvCXioLe5Na5jFsL78ra/wuBU4iuEgd4w==" }, - "node_modules/cross-fetch": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.2.0.tgz", - "integrity": "sha512-Q+xVJLoGOeIMXZmbUK4HYk+69cQH6LudR0Vu/pRm2YlU/hDV9CiS0gKUMaWY5f2NeUH9C1nV3bsTlCo0FsTV1Q==", - "dependencies": { - "node-fetch": "^2.7.0" - } - }, "node_modules/deep-object-diff": { "version": "1.1.9", "resolved": "https://registry.npmjs.org/deep-object-diff/-/deep-object-diff-1.1.9.tgz", @@ -100,19 +92,19 @@ "dev": true }, "node_modules/ignore": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.3.tgz", - "integrity": "sha512-bAH5jbK/F3T3Jls4I0SO1hmPR0dKU0a7+SY6n1yzRtG54FLO8d6w/nxLFX2Nb7dBu6cCWXPaAME6cYqFUMmuCA==", + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", "engines": { "node": ">= 4" } }, "node_modules/isbinaryfile": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/isbinaryfile/-/isbinaryfile-4.0.10.tgz", - "integrity": "sha512-iHrqe5shvBUcFbmZq9zOQHBoeOhZJu6RQGrDpBgenUm/Am+F3JM2MgQj+rK3Z601fzrL5gLZWtAPH2OBaSVcyw==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/isbinaryfile/-/isbinaryfile-5.0.6.tgz", + "integrity": "sha512-I+NmIfBHUl+r2wcDd6JwE9yWje/PIVY/R5/CmV8dXLZd5K+L9X2klAOwfAHNnondLXkbHyTAleQAWonpTJBTtw==", "engines": { - "node": ">= 8.0.0" + "node": ">= 18.0.0" }, "funding": { "url": "https://github.com/sponsors/gjtorikian/" @@ -140,183 +132,41 @@ "node": ">= 8.0.0" } }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dependencies": { - "whatwg-url": "^5.0.0" + "node_modules/prettier": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", + "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" }, "engines": { - "node": "4.x || >=6.0.0" + "node": ">=14" }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" } }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, "node_modules/typescript": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", - "integrity": "sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==", + "version": "5.9.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.2.tgz", + "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=12.20" + "node": ">=14.17" } }, "node_modules/undici-types": { - "version": "6.20.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", - "dev": true - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - } - }, - "dependencies": { - "@types/js-yaml": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", - "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", - "dev": true - }, - "@types/node": { - "version": "22.13.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.1.tgz", - "integrity": "sha512-jK8uzQlrvXqEU91UxiK5J7pKHyzgnI1Qnl0QDHIgVGuolJhRb9EEl28Cj9b3rGR8B2lhFCtvIm5os8lFnO/1Ew==", - "dev": true, - "requires": { - "undici-types": "~6.20.0" - } - }, - "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" - }, - "binary-extensions": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", - "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==" - }, - "clone": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", - "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==" - }, - "commander": { - "version": "9.5.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", - "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==" - }, - "common-path-prefix": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/common-path-prefix/-/common-path-prefix-3.0.0.tgz", - "integrity": "sha512-QE33hToZseCH3jS0qN96O/bSh3kaw/h+Tq7ngyY9eWDUnTlTNUyqfqvCXioLe5Na5jFsL78ra/wuBU4iuEgd4w==" - }, - "cross-fetch": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.2.0.tgz", - "integrity": "sha512-Q+xVJLoGOeIMXZmbUK4HYk+69cQH6LudR0Vu/pRm2YlU/hDV9CiS0gKUMaWY5f2NeUH9C1nV3bsTlCo0FsTV1Q==", - "requires": { - "node-fetch": "^2.7.0" - } - }, - "deep-object-diff": { - "version": "1.1.9", - "resolved": "https://registry.npmjs.org/deep-object-diff/-/deep-object-diff-1.1.9.tgz", - "integrity": "sha512-Rn+RuwkmkDwCi2/oXOFS9Gsr5lJZu/yTGpK7wAaAIE75CC+LCGEZHpY6VQJa/RoJcrmaA/docWJZvYohlNkWPA==", + "version": "7.10.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.10.0.tgz", + "integrity": "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==", "dev": true - }, - "ignore": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.3.tgz", - "integrity": "sha512-bAH5jbK/F3T3Jls4I0SO1hmPR0dKU0a7+SY6n1yzRtG54FLO8d6w/nxLFX2Nb7dBu6cCWXPaAME6cYqFUMmuCA==" - }, - "isbinaryfile": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/isbinaryfile/-/isbinaryfile-4.0.10.tgz", - "integrity": "sha512-iHrqe5shvBUcFbmZq9zOQHBoeOhZJu6RQGrDpBgenUm/Am+F3JM2MgQj+rK3Z601fzrL5gLZWtAPH2OBaSVcyw==" - }, - "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "requires": { - "argparse": "^2.0.1" - } - }, - "node-cache": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", - "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", - "requires": { - "clone": "2.x" - } - }, - "node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "requires": { - "whatwg-url": "^5.0.0" - } - }, - "tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "typescript": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", - "integrity": "sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==", - "dev": true - }, - "undici-types": { - "version": "6.20.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", - "dev": true - }, - "webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "requires": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } } } } diff --git a/package.json b/package.json index 732f954..46d9ae2 100644 --- a/package.json +++ b/package.json @@ -1,18 +1,19 @@ { "name": "linguist-js", - "version": "2.9.0", + "version": "3.0.0-dev", "description": "Analyse languages used in a folder. Powered by GitHub Linguist, although it doesn't need to be installed.", "main": "dist/index.js", + "type": "module", "bin": { "linguist-js": "bin/index.js", "linguist": "bin/index.js" }, "engines": { - "node": ">=12", - "npm": "<9" + "node": ">=24", + "npm": ">=10" }, "scripts": { - "download-files": "npx tsx@3 build/download-files", + "download-files": "node build/download-files", "pre-publish": "npm run download-files && npm test && npm run perf", "perf": "tsc && node test/perf", "test": "tsc && node test/folder && node test/unit" @@ -39,19 +40,19 @@ }, "homepage": "https://github.com/Nixinova/Linguist#readme", "dependencies": { - "binary-extensions": "^2.3.0 <3", - "commander": "^9.5.0 <10", + "binary-extensions": "^3.1.0", + "commander": "^14.0.0", "common-path-prefix": "^3.0.0", - "cross-fetch": "^3.2.0 <4", - "ignore": "^7.0.3", - "isbinaryfile": "^4.0.10 <5", + "ignore": "^7.0.5", + "isbinaryfile": "^5.0.6", "js-yaml": "^4.1.0", "node-cache": "^5.1.2" }, "devDependencies": { "@types/js-yaml": "^4.0.9", - "@types/node": "ts5.0", + "@types/node": "^24.3.1", "deep-object-diff": "^1.1.9", - "typescript": "~5.0.4 <5.1" + "prettier": "^3.6.2", + "typescript": "^5.9.2" } } diff --git a/readme.md b/readme.md index 5ab96ba..65dd03d 100644 --- a/readme.md +++ b/readme.md @@ -51,9 +51,8 @@ Running LinguistJS on this folder will return the following JSON: "count": 5, "bytes": 6020, "lines": { - "total": 100, - "content": 90, - "code": 80, + "total": 100, + "content": 90, }, "results": { "/src/index.ts": "TypeScript", @@ -63,57 +62,41 @@ Running LinguistJS on this folder will return the following JSON: "/x.pluginspec": "Ruby", }, "alternatives": { - "/x.pluginspec": ["XML"], + "/x.pluginspec": ["XML"], }, }, "languages": { "count": 3, "bytes": 6010, "lines": { - "total": 90, - "content": 80, - "code": 70, + "total": 90, + "content": 80, }, "results": { - "JavaScript": { - "type": "programming", - "bytes": 1000, - "lines": { "total": 49, "content": 49, "code": 44 }, - "color": "#f1e05a" - }, - "Markdown": { - "type": "prose", - "bytes": 3000, - "lines": { "total": 10, "content": 5, "code": 5 }, - "color": "#083fa1" - }, - "Ruby": { - "type": "programming", - "bytes": 10, - "lines": { "total": 1, "content": 1, "code": 1 }, - "color": "#701516" - }, - "TypeScript": { - "type": "programming", - "bytes": 2000, - "lines": { "total": 30, "content": 25, "code": 20 }, - "color": "#2b7489" - }, + "JavaScript": { "bytes": 1000, "lines": { "total": 49, "content": 49 }, }, + "Markdown": { "bytes": 3000, "lines": { "total": 10, "content": 5 }, }, + "Ruby": { "bytes": 10, "lines": { "total": 1, "content": 1 }, }, + "TypeScript": { "bytes": 2000, "lines": { "total": 30, "content": 25 }, }, }, }, "unknown": { "count": 1, "bytes": 10, "lines": { - "total": 10, - "content": 10, - "code": 10, + "total": 10, + "content": 10, }, "filenames": { "no-lang": 10, }, "extensions": {}, }, + "repository": { + "JavaScript": { "type": "programming", "color": "#f1e05a" }, + "Markdown": { "type": "prose", "color": "#083fa1" }, + "Ruby": { "type": "programming", "color": "#701516" }, + "TypeScript": { "type": "programming", "color": "#2b7489" }, + } } ``` @@ -134,13 +117,13 @@ const linguist = require('linguist-js'); // Analyse folder on disc const folder = './src'; const options = { keepVendored: false, quick: false }; -const { files, languages, unknown } = await linguist(folder, options); +const { files, languages, unknown, repository } = await linguist(folder, options); // Analyse file content from raw input const fileNames = ['file1.ts', 'file2.ts', 'ignoreme.js']; const fileContent = ['#!/usr/bin/env node', 'console.log("Example");', '"ignored"']; const options = { ignoredFiles: ['ignore*'] }; -const { files, languages, unknown } = await linguist(fileNames, { fileContent, ...options }); +const { files, languages, unknown, repository } = await linguist(fileNames, { fileContent, ...options }); ``` - `linguist(entry?, opts?)` (default export): diff --git a/src/cli.ts b/src/cli.ts index e479e13..e2e3cc5 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,20 +1,15 @@ -const VERSION = require('../package.json').version; - -import FS from 'fs'; -import Path from 'path'; import { program } from 'commander'; +import FS from 'node:fs'; +import runCliAnalysis from './cli/runCliAnalysis.js'; -import linguist from './index'; -import { normPath } from './helpers/norm-path'; - -const colouredMsg = ([r, g, b]: number[], msg: string): string => `\u001B[${38};2;${r};${g};${b}m${msg}${'\u001b[0m'}`; -const hexToRgb = (hex: string): number[] => [parseInt(hex.slice(1, 3), 16), parseInt(hex.slice(3, 5), 16), parseInt(hex.slice(5, 7), 16)]; +const packageJson = JSON.parse(FS.readFileSync(new URL('../package.json', import.meta.url), 'utf-8')); +const VERSION = packageJson.version; program .name('linguist') .usage('--analyze [] []') - .option('-a|--analyze|--analyse [folders...]', 'Analyse the languages of all files in a folder') + .option('-a|--analyze [folders...]', 'Analyse the languages of all files in a folder') .option('-i|--ignoredFiles ', `A list of file path globs to ignore`) .option('-l|--ignoredLanguages ', `A list of languages to ignore`) .option('-c|--categories ', 'Language categories to include in output') @@ -37,7 +32,7 @@ program .option('-M|--checkModeline [bool]', 'Check modelines for explicit classification', true) .helpOption(`-h|--help`, 'Display this help message') - .version(VERSION, '-v|--version', 'Display the installed version of linguist-js') + .version(VERSION, '-v|--version', 'Display the installed version of linguist-js'); program.parse(process.argv); const args = program.opts(); @@ -49,135 +44,15 @@ for (const arg in args) { val = val.replace(/^=/, ''); if (val.match(/true$|false$/)) val = val === 'true'; return val; - } + }; if (Array.isArray(args[arg])) args[arg] = args[arg].map(normalise); else args[arg] = normalise(args[arg]); } // Run Linguist -if (args.analyze) (async () => { - // Check arguments - const validCategories = ['data', 'programming', 'prose', 'markup']; - if (args.categories?.some((category: string) => !validCategories.includes(category))) { - console.log(`Error: '${args.categories.join(', ')}' contains an invalid category. Valid options: ${validCategories.join(', ')}.`); - return; - } - - // Fetch language data - const root = args.analyze === true ? '.' : args.analyze; - const data = await linguist(root, args); - const { files, languages, unknown } = data; - // Print output - if (!args.json) { - // Ignore languages with a bytes/% size less than the declared min size - if (args.minSize) { - const totalSize = languages.bytes; - const minSizeAmt = parseFloat(args.minSize.replace(/[a-z]+$/i, '')); // '2KB' -> 2 - const minSizeUnit = args.minSize.replace(/^\d+/, '').toLowerCase(); // '2KB' -> 'kb' - const checkBytes = minSizeUnit !== 'loc'; // whether to check bytes or loc - const conversionFactors: Record number> = { - 'b': n => n, - 'kb': n => n * 1e3, - 'mb': n => n * 1e6, - '%': n => n * totalSize / 100, - 'loc': n => n, - }; - const minBytesSize = conversionFactors[minSizeUnit](+minSizeAmt); - const other = { bytes: 0, lines: { total: 0, content: 0, code: 0 } }; - // Apply specified minimums: delete language results that do not reach the threshold - for (const [lang, data] of Object.entries(languages.results)) { - const checkUnit = checkBytes ? data.bytes : data.lines.code; - if (checkUnit < minBytesSize) { - // Add to 'other' count - other.bytes += data.bytes; - other.lines.total += data.lines.total; - other.lines.content += data.lines.content; - other.lines.code += data.lines.code; - // Remove language result - delete languages.results[lang]; - } - } - if (other.bytes) { - languages.results["Other"] = { ...other, type: null! }; - } - } - - const sortedEntries = Object.entries(languages.results).sort((a, b) => (a[1].bytes < b[1].bytes ? +1 : -1)); - const totalBytes = languages.bytes; - console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${files.count} files with linguist-js`); - console.log(`\n Language analysis results: \n`); - let count = 0; - if (sortedEntries.length === 0) console.log(` None`); - - // Collate files per language - const filesPerLanguage: Record = {}; - if (args.listFiles) { - for (const language of Object.keys(languages.results)) { - filesPerLanguage[language] = []; - } - for (const [file, lang] of Object.entries(files.results)) { - if (lang) - filesPerLanguage[lang].push(file); - } - } - // List parsed results - for (const [lang, { bytes, lines, color }] of sortedEntries) { - const percent = (bytes: number) => bytes / (totalBytes || 1) * 100; - const fmtd = { - index: (++count).toString().padStart(2, ' '), - lang: lang.padEnd(24, ' '), - percent: percent(bytes).toFixed(2).padStart(5, ' '), - bytes: bytes.toLocaleString().padStart(10, ' '), - loc: lines.code.toLocaleString().padStart(10, ' '), - icon: colouredMsg(hexToRgb(color ?? '#ededed'), '\u2588'), - }; - console.log(` ${fmtd.index}. ${fmtd.icon} ${fmtd.lang} ${fmtd.percent}% ${fmtd.bytes} B ${fmtd.loc} LOC`); - - // If using `listFiles` option, list all files tagged as this language - if (args.listFiles) { - console.log(); // padding - for (const file of filesPerLanguage[lang]) { - let relFile = normPath(Path.relative(Path.resolve('.'), file)); - if (!relFile.startsWith('../')) relFile = './' + relFile; - const bytes = (await FS.promises.stat(file)).size; - const fmtd2 = { - file: relFile.padEnd(42, ' '), - percent: percent(bytes).toFixed(2).padStart(5, ' '), - bytes: bytes.toLocaleString().padStart(10, ' '), - } - console.log(` ${fmtd.icon} ${fmtd2.file} ${fmtd2.percent}% ${fmtd2.bytes} B`); - } - console.log(); // padding - } - } - if (!args.listFiles) console.log(); // padding - console.log(` Total: ${totalBytes.toLocaleString()} B`); - // List unknown files/extensions - if (unknown.bytes > 0) { - console.log(`\n Unknown files and extensions:`); - for (const [name, bytes] of Object.entries(unknown.filenames)) { - console.log(` '${name}': ${bytes.toLocaleString()} B`); - } - for (const [ext, bytes] of Object.entries(unknown.extensions)) { - console.log(` '*${ext}': ${bytes.toLocaleString()} B`); - } - console.log(` Total: ${unknown.bytes.toLocaleString()} B`); - } - } - else if (args.tree) { - const treeParts: string[] = args.tree.split('.'); - let nestedData: Record = data; - for (const part of treeParts) { - if (!nestedData[part]) throw Error(`TraversalError: Key '${part}' cannot be found on output object.`); - nestedData = nestedData[part]; - } - console.log(nestedData); - } - else { - console.dir(data, { depth: null }); - } -})(); -else { +if (args.analyze) { + void runCliAnalysis(args); +} else { console.log(`Welcome to linguist-js, a JavaScript port of GitHub's language analyzer.`); console.log(`Type 'linguist --help' for a list of commands.`); } diff --git a/src/cli/output/default.ts b/src/cli/output/default.ts new file mode 100644 index 0000000..bb50a7b --- /dev/null +++ b/src/cli/output/default.ts @@ -0,0 +1,113 @@ +import { OptionValues } from 'commander'; +import FS from 'node:fs'; +import Path from 'node:path'; +import { normPath } from '../../program/fs/normalisedPath.js'; +import { Results } from '../../types/types.js'; +import { colouredMsg, hexToRgb } from '../utils.js'; + +export default async function defaultOutput(args: OptionValues, data: Results) { + const { files, languages, unknown, repository } = data; + + if (args.minSize) { + // Ignore languages with a bytes/% size less than the declared min size + + const totalSize = languages.bytes; + const minSizeAmt = parseFloat(args.minSize.replace(/[a-z]+$/i, '')); // '2KB' -> 2 + const minSizeUnit = args.minSize.replace(/^\d+/, '').toLowerCase(); // '2KB' -> 'kb' + const checkBytes = minSizeUnit !== 'loc'; // whether to check bytes or loc + const conversionFactors: Record number> = { + ['b']: (n) => n, + ['kb']: (n) => n * 1e3, + ['mb']: (n) => n * 1e6, + ['%']: (n) => (n * totalSize) / 100, + ['loc']: (n) => n, + }; + const minBytesSize = conversionFactors[minSizeUnit](+minSizeAmt); + const other = { count: 0, bytes: 0, lines: { total: 0, content: 0, code: 0 } }; + // Apply specified minimums: delete language results that do not reach the threshold + for (const [lang, data] of Object.entries(languages.results)) { + const checkUnit = checkBytes ? data.bytes : data.lines.content; + if (checkUnit < minBytesSize) { + // Add to 'other' count + other.count++; + other.bytes += data.bytes; + other.lines.total += data.lines.total; + other.lines.content += data.lines.content; + // Remove language result + delete languages.results[lang]; + } + } + if (other.bytes) { + languages.results['Other'] = other; + } + } + + const sortedEntries = Object.entries(languages.results).sort((a, b) => (a[1].bytes < b[1].bytes ? +1 : -1)); + const totalBytes = languages.bytes; + console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${files.count} files with linguist-js`); + console.log(`\n Language analysis results: \n`); + let count = 0; + if (sortedEntries.length === 0) console.log(` None`); + + // Collate files per language + const filesPerLanguage: Record = {}; + if (args.listFiles) { + for (const language of Object.keys(languages.results)) { + filesPerLanguage[language] = []; + } + for (const [file, lang] of Object.entries(files.results)) { + if (lang) { + filesPerLanguage[lang].push(file); + } + } + } + // List parsed results + for (const [lang, { bytes, lines }] of sortedEntries) { + const colour = hexToRgb(repository[lang].color ?? '#ededed'); + const percent = (bytes: number) => (bytes / (totalBytes || 1)) * 100; + const fmtd = { + index: (++count).toString().padStart(2, ' '), + lang: lang.padEnd(24, ' '), + percent: percent(bytes).toFixed(2).padStart(5, ' '), + bytes: bytes.toLocaleString().padStart(10, ' '), + loc: lines.content.toLocaleString().padStart(10, ' '), + icon: colouredMsg(colour, '\u2588'), + }; + console.log(` ${fmtd.index}. ${fmtd.icon} ${fmtd.lang} ${fmtd.percent}% ${fmtd.bytes} B ${fmtd.loc} LOC`); + + // If using `listFiles` option, list all files tagged as this language + if (args.listFiles) { + console.log(); // padding + for (const file of filesPerLanguage[lang]) { + let relFile = normPath(Path.relative(Path.resolve('.'), file)); + if (!relFile.startsWith('../')) { + relFile = './' + relFile; + } + const fileStat = await FS.promises.stat(file); + const bytes = fileStat.size; + const fmtd2 = { + file: relFile.padEnd(42, ' '), + percent: percent(bytes).toFixed(2).padStart(5, ' '), + bytes: bytes.toLocaleString().padStart(10, ' '), + }; + console.log(` ${fmtd.icon} ${fmtd2.file} ${fmtd2.percent}% ${fmtd2.bytes} B`); + } + console.log(); // padding + } + } + if (!args.listFiles) { + console.log(); // padding + } + console.log(` Total: ${totalBytes.toLocaleString()} B`); + // List unknown files/extensions + if (unknown.bytes > 0) { + console.log(`\n Unknown files and extensions:`); + for (const [name, bytes] of Object.entries(unknown.filenames)) { + console.log(` '${name}': ${bytes.toLocaleString()} B`); + } + for (const [ext, bytes] of Object.entries(unknown.extensions)) { + console.log(` '*${ext}': ${bytes.toLocaleString()} B`); + } + console.log(` Total: ${unknown.bytes.toLocaleString()} B`); + } +} diff --git a/src/cli/output/tree.ts b/src/cli/output/tree.ts new file mode 100644 index 0000000..da61761 --- /dev/null +++ b/src/cli/output/tree.ts @@ -0,0 +1,14 @@ +import { OptionValues } from 'commander'; +import { Results } from '../../types/types.js'; + +export default function treeOutput(args: OptionValues, data: Results) { + const treeParts: string[] = args.tree.split('.'); + let nestedData: Record = data; + for (const part of treeParts) { + if (!nestedData[part]) { + throw Error(`TraversalError: Key '${part}' cannot be found on output object.`); + } + nestedData = nestedData[part]; + } + console.log(nestedData); +} diff --git a/src/cli/runCliAnalysis.ts b/src/cli/runCliAnalysis.ts new file mode 100644 index 0000000..f6bbca1 --- /dev/null +++ b/src/cli/runCliAnalysis.ts @@ -0,0 +1,28 @@ +import { OptionValues } from 'commander'; +import linguist from '../index.js'; +import defaultOutput from './output/default.js'; +import treeOutput from './output/tree.js'; + +const validCategories = ['data', 'programming', 'prose', 'markup']; + +export default async function runCliAnalysis(args: OptionValues) { + // Check arguments + if (args.categories?.some((category: string) => !validCategories.includes(category))) { + console.log(`Error: '${args.categories.join(', ')}' contains an invalid category.`); + console.log(`\tValid options: ${validCategories.join(', ')}.`); + return; + } + + // Fetch language data + const root = args.analyze === true ? '.' : args.analyze; + const data = await linguist(root, args); + + // Print output + if (!args.json) { + defaultOutput(args, data); + } else if (args.tree) { + treeOutput(args, data); + } else { + console.dir(data, { depth: null }); + } +} diff --git a/src/cli/utils.ts b/src/cli/utils.ts new file mode 100644 index 0000000..6a2e80f --- /dev/null +++ b/src/cli/utils.ts @@ -0,0 +1,10 @@ +export function colouredMsg([r, g, b]: [number, number, number], msg: string): string { + return `\u001B[${38};2;${r};${g};${b}m${msg}\u001b[0m`; +} + +export function hexToRgb(hex: string): [number, number, number] { + const r = parseInt(hex.slice(1, 3), 16); + const g = parseInt(hex.slice(3, 5), 16); + const b = parseInt(hex.slice(5, 7), 16); + return [r, g, b]; +} diff --git a/src/index.ts b/src/index.ts index 2513141..22a6c9c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,26 +1,35 @@ -import FS from 'fs'; -import Path from 'path'; -import YAML from 'js-yaml'; -import ignore, { Ignore } from 'ignore'; import commonPrefix from 'common-path-prefix'; -import binaryData from 'binary-extensions'; +import ignore, { Ignore } from 'ignore'; import { isBinaryFile } from 'isbinaryfile'; +import FS from 'node:fs'; +import Path from 'node:path'; +import Attributes from './program/classes/attributes.js'; +import retrieveData from './program/data/retrieveData.js'; +import { normPath } from './program/fs/normalisedPath.js'; +import readFileChunk from './program/fs/readFile.js'; +import walkTree from './program/fs/walkTree.js'; +import parseGitattributes from './program/parsing/parseGitattributes.js'; +import pcre from './program/utils/pcre.js'; +import * as T from './types/types.js'; + +const binaryData = JSON.parse( + FS.readFileSync(new URL('../node_modules/binary-extensions/binary-extensions.json', import.meta.url), 'utf-8') +) as string[]; + +async function analyse(path?: string, opts?: T.Options): Promise; +async function analyse(paths?: string[], opts?: T.Options): Promise; +async function analyse(content?: Record, opts?: T.Options): Promise; +async function analyse(rawInput?: string | string[] | Record, opts: T.Options = {}): Promise { + const inputs = { + path: typeof rawInput === 'string' ? rawInput : null, + paths: Array.isArray(rawInput) ? rawInput : null, + content: typeof rawInput === 'object' && !Array.isArray(rawInput) ? rawInput : null, + }; + const inputPaths = inputs.paths ?? (inputs.path ? [inputs.path] : null); + const inputContent = inputs.content; + const useRawContent = inputContent !== null; -import walk from './helpers/walk-tree'; -import loadFile, { parseGeneratedDataFile } from './helpers/load-data'; -import readFileChunk from './helpers/read-file'; -import parseAttributes, { FlagAttributes } from './helpers/parse-gitattributes'; -import pcre from './helpers/convert-pcre'; -import { normPath } from './helpers/norm-path'; -import * as T from './types'; -import * as S from './schema'; - -async function analyse(path?: string, opts?: T.Options): Promise -async function analyse(paths?: string[], opts?: T.Options): Promise -async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Promise { - const useRawContent = opts.fileContent !== undefined; - const input = [rawPaths ?? []].flat(); - const manualFileContent = [opts.fileContent ?? []].flat(); + const input = useRawContent ? Object.keys(inputContent) : (inputPaths ?? []); // Normalise input option arguments opts = { @@ -35,28 +44,24 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom }; // Load data from github-linguist web repo - const langData = await loadFile('languages.yml', opts.offline).then(YAML.load); - const vendorData = await loadFile('vendor.yml', opts.offline).then(YAML.load); - const docData = await loadFile('documentation.yml', opts.offline).then(YAML.load); - const heuristicsData = await loadFile('heuristics.yml', opts.offline).then(YAML.load); - const generatedData = await loadFile('generated.rb', opts.offline).then(parseGeneratedDataFile); - const vendorPaths = [...vendorData, ...docData, ...generatedData]; + const { langData, heuristicsData, vendorPaths } = await retrieveData(opts.offline ?? false); // Setup main variables const fileAssociations: Record = {}; const extensions: Record = {}; const globOverrides: Record = {}; const results: T.Results = { - files: { count: 0, bytes: 0, lines: { total: 0, content: 0, code: 0 }, results: {}, alternatives: {} }, - languages: { count: 0, bytes: 0, lines: { total: 0, content: 0, code: 0 }, results: {} }, - unknown: { count: 0, bytes: 0, lines: { total: 0, content: 0, code: 0 }, extensions: {}, filenames: {} }, + files: { count: 0, bytes: 0, lines: { total: 0, content: 0 }, results: {}, alternatives: {} }, + languages: { count: 0, bytes: 0, lines: { total: 0, content: 0 }, results: {} }, + unknown: { count: 0, bytes: 0, lines: { total: 0, content: 0 }, extensions: {}, filenames: {} }, + repository: {}, }; // Set a common root path so that vendor paths do not incorrectly match parent folders - const resolvedInput = input.map(path => normPath(Path.resolve(path))); + const resolvedInput = input.map((path) => normPath(Path.resolve(path))); const commonRoot = (input.length > 1 ? commonPrefix(resolvedInput) : resolvedInput[0]).replace(/\/?$/, ''); - const relPath = (file: T.AbsFile): T.RelFile => useRawContent ? file : normPath(Path.relative(commonRoot, file)); - const unRelPath = (file: T.RelFile): T.AbsFile => useRawContent ? file : normPath(Path.resolve(commonRoot, file)); + const relPath = (file: T.AbsFile): T.RelFile => (useRawContent ? file : normPath(Path.relative(commonRoot, file))); + const unRelPath = (file: T.RelFile): T.AbsFile => (useRawContent ? file : normPath(Path.resolve(commonRoot, file))); // Other helper functions const fileMatchesGlobs = (file: T.AbsFile, ...globs: T.FileGlob[]) => ignore().add(globs).ignores(relPath(file)); @@ -68,50 +73,30 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom const ignored = ignore(); ignored.add('.git/'); ignored.add(opts.ignoredFiles ?? []); - const regexIgnores: RegExp[] = opts.keepVendored ? [] : vendorPaths.map(path => RegExp(path, 'i')); + const regexIgnores: RegExp[] = opts.keepVendored ? [] : vendorPaths.map((path) => RegExp(path, 'i')); // Load file paths and folders let files: T.AbsFile[]; if (useRawContent) { // Uses raw file content files = input; - } - else { + } else { // Uses directory on disc - const data = walk({ init: true, commonRoot, folderRoots: resolvedInput, folders: resolvedInput, ignored }); + const data = walkTree({ init: true, commonRoot, folderRoots: resolvedInput, folders: resolvedInput, ignored }); files = data.files; } // Fetch and normalise gitattributes data of all subfolders and save to metadata - const manualAttributes: Record = {}; // Maps file globs to gitattribute boolean flags - const getFlaggedGlobs = (attr: keyof FlagAttributes, val: boolean) => { - return Object.entries(manualAttributes).filter(([, attrs]) => attrs[attr] === val).map(([glob,]) => glob) - }; - const findAttrsForPath = (filePath: string): FlagAttributes | null => { - const resultAttrs: Record = {}; - for (const glob in manualAttributes) { - if (ignore().add(glob).ignores(relPath(filePath))) { - const matchingAttrs = manualAttributes[glob]; - for (const [attr, val] of Object.entries(matchingAttrs)) { - if (val !== null) resultAttrs[attr] = val; - } - } - } - - if (!JSON.stringify(resultAttrs)) { - return null; - } - return resultAttrs as FlagAttributes; - } + const manualAttributes = new Attributes(); if (!useRawContent && opts.checkAttributes) { - const nestedAttrFiles = files.filter(file => file.endsWith('.gitattributes')); + const nestedAttrFiles = files.filter((file) => file.endsWith('.gitattributes')); for (const attrFile of nestedAttrFiles) { const relAttrFile = relPath(attrFile); const relAttrFolder = Path.dirname(relAttrFile); const contents = await readFileChunk(attrFile); - const parsed = parseAttributes(contents, relAttrFolder); + const parsed = parseGitattributes(contents, relAttrFolder); for (const { glob, attrs } of parsed) { - manualAttributes[glob] = attrs; + manualAttributes.add(glob, attrs); } } } @@ -121,13 +106,13 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom for (const file of files) { const relFile = relPath(file); - const isRegexIgnored = regexIgnores.some(pattern => pattern.test(relFile)); + const isRegexIgnored = regexIgnores.some((pattern) => pattern.test(relFile)); if (!isRegexIgnored) { // Checking overrides is moot if file is not even marked as ignored by default continue; } - const fileAttrs = findAttrsForPath(file); + const fileAttrs = manualAttributes.findAttrsForPath(relPath(file)); if (fileAttrs?.generated === false || fileAttrs?.vendored === false) { // File is explicitly marked as *not* to be ignored // do nothing @@ -135,35 +120,43 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom filesToIgnore.push(file); } } - files = files.filter(file => !filesToIgnore.includes(file)); + files = files.filter((file) => !filesToIgnore.includes(file)); // Apply vendor file path matches and filter out vendored files if (!opts.keepVendored) { // Get data of files that have been manually marked with metadata - const vendorTrueGlobs = [...getFlaggedGlobs('vendored', true), ...getFlaggedGlobs('generated', true), ...getFlaggedGlobs('documentation', true)]; - const vendorFalseGlobs = [...getFlaggedGlobs('vendored', false), ...getFlaggedGlobs('generated', false), ...getFlaggedGlobs('documentation', false)]; + const vendorTrueGlobs = [ + ...manualAttributes.getFlaggedGlobs('vendored', true), + ...manualAttributes.getFlaggedGlobs('generated', true), + ...manualAttributes.getFlaggedGlobs('documentation', true), + ]; + const vendorFalseGlobs = [ + ...manualAttributes.getFlaggedGlobs('vendored', false), + ...manualAttributes.getFlaggedGlobs('generated', false), + ...manualAttributes.getFlaggedGlobs('documentation', false), + ]; // Set up glob ignore object to use for expanding globs to match files const vendorTrueIgnore = ignore().add(vendorTrueGlobs); const vendorFalseIgnore = ignore().add(vendorFalseGlobs); // Remove all files marked as vendored by default - const excludedFiles = files.filter(file => vendorPaths.some(pathPtn => RegExp(pathPtn, 'i').test(relPath(file)))); - files = files.filter(file => !excludedFiles.includes(file)); + const excludedFiles = files.filter((file) => vendorPaths.some((pathPtn) => RegExp(pathPtn, 'i').test(relPath(file)))); + files = files.filter((file) => !excludedFiles.includes(file)); // Re-add removed files that are overridden manually in gitattributes - const overriddenExcludedFiles = excludedFiles.filter(file => vendorFalseIgnore.ignores(relPath(file))); + const overriddenExcludedFiles = excludedFiles.filter((file) => vendorFalseIgnore.ignores(relPath(file))); files.push(...overriddenExcludedFiles); // Remove files explicitly marked as vendored in gitattributes - files = files.filter(file => !vendorTrueIgnore.ignores(relPath(file))); + files = files.filter((file) => !vendorTrueIgnore.ignores(relPath(file))); } // Filter out binary files if (!opts.keepBinary) { // Filter out files that are binary by default - files = files.filter(file => !binaryData.some(ext => file.endsWith('.' + ext))); + files = files.filter((file) => !binaryData.some((ext) => file.endsWith('.' + ext))); // Filter out manually specified binary files - const binaryIgnored = ignore().add(getFlaggedGlobs('binary', true)); + const binaryIgnored = ignore().add(manualAttributes.getFlaggedGlobs('binary', true)); files = filterOutIgnored(files, binaryIgnored); // Re-add files manually marked not as binary - const binaryUnignored = ignore().add(getFlaggedGlobs('binary', false)); + const binaryUnignored = ignore().add(manualAttributes.getFlaggedGlobs('binary', false)); const unignoredList = filterOutIgnored(files, binaryUnignored); files.push(...unignoredList); } @@ -186,7 +179,7 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom // If specified language is an alias, associate it with its full name if (!langData[forcedLang]) { - const overrideLang = Object.entries(langData).find(entry => entry[1].aliases?.includes(forcedLang!.toLowerCase())); + const overrideLang = Object.entries(langData).find((entry) => entry[1].aliases?.includes(forcedLang!.toLowerCase())); if (overrideLang) { forcedLang = overrideLang[0]; } @@ -203,7 +196,7 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom } // Set parent to result group if it is present // Is nullish if either `opts.childLanguages` is set or if there is no group - const finalResult = !opts.childLanguages && result && langData[result] && langData[result].group || result; + const finalResult = (!opts.childLanguages && result && langData[result] && langData[result].group) || result; if (!fileAssociations[file].includes(finalResult)) { fileAssociations[file].push(finalResult); } @@ -213,8 +206,7 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom const definiteness: Record = {}; const fromShebang: Record = {}; - fileLoop: - for (const file of files) { + fileLoop: for (const file of files) { // Check manual override for (const globMatch in globOverrides) { if (!fileMatchesGlobs(file, globMatch)) continue; @@ -229,44 +221,39 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom // Check first line for readability let firstLine: string | null; if (useRawContent) { - firstLine = manualFileContent[files.indexOf(file)]?.split('\n')[0] ?? null; - } - else if (FS.existsSync(file) && !FS.lstatSync(file).isDirectory()) { + firstLine = inputContent[file]?.split('\n')[0] ?? null; + } else if (FS.existsSync(file) && !FS.lstatSync(file).isDirectory()) { firstLine = await readFileChunk(file, true).catch(() => null); - } - else continue; + } else continue; // Skip if file is unreadable or blank if (firstLine === null) continue; // Check first line for explicit classification + const modelineRegex = /-\*-|(?:syntax|filetype|ft)\s*=/; const hasShebang = opts.checkShebang && /^#!/.test(firstLine); - const hasModeline = opts.checkModeline && /-\*-|(syntax|filetype|ft)\s*=/.test(firstLine); + const hasModeline = opts.checkModeline && modelineRegex.test(firstLine); if (!opts.quick && (hasShebang || hasModeline)) { const matches = []; for (const [lang, data] of Object.entries(langData)) { const langMatcher = (lang: string) => `\\b${lang.toLowerCase().replace(/\W/g, '\\$&')}(?![\\w#+*]|-\*-)`; // Check for interpreter match if (opts.checkShebang && hasShebang) { - const matchesInterpretor = data.interpreters?.some(interpreter => firstLine!.match(`\\b${interpreter}\\b`)); - if (matchesInterpretor) - matches.push(lang); + const matchesInterpretor = data.interpreters?.some((interpreter) => firstLine.match(`\\b${interpreter}\\b`)); + if (matchesInterpretor) matches.push(lang); } // Check modeline declaration if (opts.checkModeline && hasModeline) { - const modelineText = firstLine!.toLowerCase().replace(/^.*-\*-(.+)-\*-.*$/, '$1'); + const modelineText = firstLine.toLowerCase().split(modelineRegex)[1]; const matchesLang = modelineText.match(langMatcher(lang)); - const matchesAlias = data.aliases?.some(lang => modelineText.match(langMatcher(lang))); - if (matchesLang || matchesAlias) - matches.push(lang); + const matchesAlias = data.aliases?.some((lang) => modelineText.match(langMatcher(lang))); + if (matchesLang || matchesAlias) matches.push(lang); } } // Add identified language(s) if (matches.length) { - for (const match of matches) - addResult(file, match); - if (matches.length === 1) - definiteness[file] = true; + for (const match of matches) addResult(file, match); + if (matches.length === 1) definiteness[file] = true; fromShebang[file] = true; continue; } @@ -275,24 +262,24 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom let skipExts = false; // Check if filename is a match for (const lang in langData) { - const matchesName = langData[lang].filenames?.some(name => Path.basename(file.toLowerCase()) === name.toLowerCase()); + const matchesName = langData[lang].filenames?.some((name) => Path.basename(file.toLowerCase()) === name.toLowerCase()); if (matchesName) { addResult(file, lang); skipExts = true; } } // Check if extension is a match - const possibleExts: { ext: string, lang: T.Language }[] = []; - if (!skipExts) for (const lang in langData) { - const extMatches = langData[lang].extensions?.filter(ext => file.toLowerCase().endsWith(ext.toLowerCase())); - if (extMatches?.length) { - for (const ext of extMatches) - possibleExts.push({ ext, lang }); + const possibleExts: { ext: string; lang: T.Language }[] = []; + if (!skipExts) + for (const lang in langData) { + const extMatches = langData[lang].extensions?.filter((ext) => file.toLowerCase().endsWith(ext.toLowerCase())); + if (extMatches?.length) { + for (const ext of extMatches) possibleExts.push({ ext, lang }); + } } - } // Apply more specific extension if available const isComplexExt = (ext: string) => /\..+\./.test(ext); - const hasComplexExt = possibleExts.some(data => isComplexExt(data.ext)); + const hasComplexExt = possibleExts.some((data) => isComplexExt(data.ext)); for (const { ext, lang } of possibleExts) { if (hasComplexExt && !isComplexExt(ext)) continue; if (!hasComplexExt && isComplexExt(ext)) continue; @@ -317,83 +304,78 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom } // Parse heuristics if applicable - if (opts.checkHeuristics) for (const heuristics of heuristicsData.disambiguations) { - // Make sure the extension matches the current file - if (!fromShebang[file] && !heuristics.extensions.includes(extensions[file])) - continue; - // Load heuristic rules - for (const heuristic of heuristics.rules) { - // Make sure the language is not an array - if (Array.isArray(heuristic.language)) { - heuristic.language = heuristic.language[0]; - } + if (opts.checkHeuristics) + for (const heuristics of heuristicsData.disambiguations) { + // Make sure the extension matches the current file + if (!fromShebang[file] && !heuristics.extensions.includes(extensions[file])) continue; + // Load heuristic rules + for (const heuristic of heuristics.rules) { + // Make sure the language is not an array + if (Array.isArray(heuristic.language)) { + heuristic.language = heuristic.language[0]; + } - // Make sure the results includes this language - const languageGroup = langData[heuristic.language]?.group; - const matchesLang = fileAssociations[file].includes(heuristic.language); - const matchesParent = languageGroup && fileAssociations[file].includes(languageGroup); - if (!matchesLang && !matchesParent) - continue; - - // Normalise heuristic data - const patterns: string[] = []; - const normalise = (contents: string | string[]) => patterns.push(...[contents].flat()); - if (heuristic.pattern) normalise(heuristic.pattern); - if (heuristic.named_pattern) normalise(heuristicsData.named_patterns[heuristic.named_pattern]); - if (heuristic.and) { - for (const data of heuristic.and) { - if (data.pattern) normalise(data.pattern); - if (data.named_pattern) normalise(heuristicsData.named_patterns[data.named_pattern]); + // Make sure the results includes this language + const languageGroup = langData[heuristic.language]?.group; + const matchesLang = fileAssociations[file].includes(heuristic.language); + const matchesParent = languageGroup && fileAssociations[file].includes(languageGroup); + if (!matchesLang && !matchesParent) continue; + + // Normalise heuristic data + const patterns: string[] = []; + const normalise = (contents: string | string[]) => patterns.push(...[contents].flat()); + if (heuristic.pattern) normalise(heuristic.pattern); + if (heuristic.named_pattern) normalise(heuristicsData.named_patterns[heuristic.named_pattern]); + if (heuristic.and) { + for (const data of heuristic.and) { + if (data.pattern) normalise(data.pattern); + if (data.named_pattern) normalise(heuristicsData.named_patterns[data.named_pattern]); + } } - } - // Check file contents and apply heuristic patterns - const fileContent = opts.fileContent ? manualFileContent[files.indexOf(file)] : await readFileChunk(file).catch(() => null); + // Check file contents and apply heuristic patterns + const fileContent = useRawContent ? inputContent[file] : await readFileChunk(file).catch(() => null); - // Skip if file read errors - if (fileContent === null) continue; + // Skip if file read errors + if (fileContent === null) continue; - // Apply heuristics - if (!patterns.length || patterns.some(pattern => pcre(pattern).test(fileContent))) { - results.files.results[file] = heuristic.language; - break; + // Apply heuristics + if (!patterns.length || patterns.some((pattern) => pcre(pattern).test(fileContent))) { + results.files.results[file] = heuristic.language; + break; + } } } - } // If no heuristics, assign a language if (!results.files.results[file]) { const possibleLangs = fileAssociations[file]; // Assign first language as a default option const defaultLang = possibleLangs[0]; - const alternativeLangs = possibleLangs.slice(1) + const alternativeLangs = possibleLangs.slice(1); results.files.results[file] = defaultLang; // List alternative languages if there are any - if (alternativeLangs.length > 0) - results.files.alternatives[file] = alternativeLangs; + if (alternativeLangs.length > 0) results.files.alternatives[file] = alternativeLangs; } } // Skip specified categories if (opts.categories?.length) { const categories: T.Category[] = ['data', 'markup', 'programming', 'prose']; - const hiddenCategories = categories.filter(cat => !opts.categories!.includes(cat)); + const hiddenCategories = categories.filter((cat) => !opts.categories!.includes(cat)); for (const [file, lang] of Object.entries(results.files.results)) { // Skip if language is not hidden - if (!hiddenCategories.some(cat => lang && langData[lang]?.type === cat)) - continue; + if (!hiddenCategories.some((cat) => lang && langData[lang]?.type === cat)) continue; // Skip if language is forced as detectable if (opts.checkDetected) { - const detectable = ignore().add(getFlaggedGlobs('detectable', true)); - if (detectable.ignores(relPath(file))) - continue; + const detectable = ignore().add(manualAttributes.getFlaggedGlobs('detectable', true)); + if (detectable.ignores(relPath(file))) continue; } // Delete result otherwise delete results.files.results[file]; - if (lang) - delete results.languages.results[lang]; + if (lang) delete results.languages.results[lang]; } for (const category of hiddenCategories) { - for (const [lang, { type }] of Object.entries(results.languages.results)) { + for (const [lang, { type }] of Object.entries(results.repository)) { if (type === category) { delete results.languages.results[lang]; } @@ -418,43 +400,40 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom for (const [file, lang] of Object.entries(results.files.results)) { if (lang && !langData[lang]) continue; // Calculate file size - const fileSize = manualFileContent[files.indexOf(file)]?.length ?? FS.statSync(file).size; + const fileSize = useRawContent ? inputContent[file]?.length : FS.statSync(file).size; // Calculate lines of code - const loc = { total: 0, content: 0, code: 0 }; + const loc = { total: 0, content: 0 }; if (opts.calculateLines) { - const fileContent = (manualFileContent[files.indexOf(file)] ?? FS.readFileSync(file).toString()) ?? ''; + const fileContent = useRawContent ? inputContent[file] : FS.readFileSync(file).toString(); const allLines = fileContent.split(/\r?\n/gm); loc.total = allLines.length; - loc.content = allLines.filter(line => line.trim().length > 0).length; - const codeLines = fileContent - .replace(/^\s*(\/\/|# |;|--).+/gm, '') - .replace(/\/\*.+\*\/|/sg, '') - loc.code = codeLines.split(/\r?\n/gm).filter(line => line.trim().length > 0).length; + loc.content = allLines.filter((line) => line.trim().length > 0).length; } // Apply to files totals results.files.bytes += fileSize; results.files.lines.total += loc.total; results.files.lines.content += loc.content; - results.files.lines.code += loc.code; // Add results to 'languages' section if language match found, or 'unknown' section otherwise if (lang) { - const { type } = langData[lang]; + // update language in repository if not yet present + if (!results.repository[lang]) { + const { type, color } = langData[lang]; + results.repository[lang] = { type, color }; + if (opts.childLanguages) { + results.repository[lang].parent = langData[lang].group; + } + } // set default if unset - results.languages.results[lang] ??= { type, bytes: 0, lines: { total: 0, content: 0, code: 0 }, color: langData[lang].color }; + results.languages.results[lang] ??= { count: 0, bytes: 0, lines: { total: 0, content: 0 } }; // apply results to 'languages' section - if (opts.childLanguages) { - results.languages.results[lang].parent = langData[lang].group; - } + results.languages.results[lang].count++; results.languages.results[lang].bytes += fileSize; results.languages.bytes += fileSize; results.languages.results[lang].lines.total += loc.total; results.languages.results[lang].lines.content += loc.content; - results.languages.results[lang].lines.code += loc.code; results.languages.lines.total += loc.total; results.languages.lines.content += loc.content; - results.languages.lines.code += loc.code; - } - else { + } else { const ext = Path.extname(file); const unknownType = ext ? 'extensions' : 'filenames'; const name = ext || Path.basename(file); @@ -464,13 +443,12 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom results.unknown.bytes += fileSize; results.unknown.lines.total += loc.total; results.unknown.lines.content += loc.content; - results.unknown.lines.code += loc.code; } } // Set lines output to NaN when line calculation is disabled if (opts.calculateLines === false) { - results.files.lines = { total: NaN, content: NaN, code: NaN } + results.files.lines = { total: NaN, content: NaN }; } // Set counts @@ -481,4 +459,4 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom // Return return results; } -export = analyse; +export default analyse; diff --git a/src/program/classes/attributes.ts b/src/program/classes/attributes.ts new file mode 100644 index 0000000..5fd7091 --- /dev/null +++ b/src/program/classes/attributes.ts @@ -0,0 +1,46 @@ +import ignore from 'ignore'; +import { FileGlob, RelFile } from '../../types/types.js'; +import { FlagAttributes } from '../parsing/parseGitattributes.js'; + +/** Stores parsed attribute information per file glob */ +export default class Attributes { + #attributes: Record; + + constructor() { + this.#attributes = {}; + } + + get attributes() { + return this.#attributes; + } + + add(glob: FileGlob, attributes: FlagAttributes) { + this.#attributes[glob] = attributes; + } + + getFlaggedGlobs(attr: keyof FlagAttributes, val: boolean) { + return Object.entries(this.#attributes) + .filter(([, attrs]) => attrs[attr] === val) + .map(([glob]) => glob); + } + + findAttrsForPath(relFilePath: RelFile): FlagAttributes | null { + const resultAttrs: Record = {}; + for (const glob in this.#attributes) { + const matchingAttrs = this.#attributes[glob]; + // Check if glob matches rel path + if (ignore().add(glob).ignores(relFilePath)) { + for (const [attr, val] of Object.entries(matchingAttrs)) { + if (val !== null) { + resultAttrs[attr] = val; + } + } + } + } + + if (!JSON.stringify(resultAttrs).length) { + return null; + } + return resultAttrs as FlagAttributes; + } +} diff --git a/src/helpers/load-data.ts b/src/program/data/loadDataFiles.ts similarity index 58% rename from src/helpers/load-data.ts rename to src/program/data/loadDataFiles.ts index 8b27a1a..5568d0e 100644 --- a/src/helpers/load-data.ts +++ b/src/program/data/loadDataFiles.ts @@ -1,9 +1,10 @@ -import FS from 'fs'; -import Path from 'path'; -import fetch from 'cross-fetch'; import Cache from 'node-cache'; +import FS from 'node:fs'; +import Path from 'node:path'; +import { fileURLToPath } from 'node:url'; const cache = new Cache({}); +const dirname = Path.dirname(fileURLToPath(import.meta.url)); async function loadWebFile(file: string): Promise { // Return cache if it exists @@ -12,22 +13,24 @@ async function loadWebFile(file: string): Promise { // Otherwise cache the request const dataUrl = (file: string): string => `https://raw.githubusercontent.com/github/linguist/HEAD/lib/linguist/${file}`; // Load file content, falling back to the local file if the request fails - const fileContent = await fetch(dataUrl(file)).then(data => data.text()).catch(async () => await loadLocalFile(file)); + const fileContent = await fetch(dataUrl(file)) + .then((data) => data.text()) + .catch(async () => await loadLocalFile(file)); cache.set(file, fileContent); return fileContent; } async function loadLocalFile(file: string): Promise { - const filePath = Path.resolve(__dirname, '../../ext', file); - return FS.promises.readFile(filePath).then(buffer => buffer.toString()); + const filePath = Path.resolve(dirname, '../../ext', file); + return FS.promises.readFile(filePath).then((buffer) => buffer.toString()); } /** Nukes unused `generated.rb` file content. */ export function parseGeneratedDataFile(fileContent: string): string[] { - return [...fileContent.match(/(?<=name\.match\(\/).+?(?=(? { +export function loadFile(file: string, offline: boolean = false): Promise { return offline ? loadLocalFile(file) : loadWebFile(file); } diff --git a/src/program/data/retrieveData.ts b/src/program/data/retrieveData.ts new file mode 100644 index 0000000..cc91906 --- /dev/null +++ b/src/program/data/retrieveData.ts @@ -0,0 +1,41 @@ +import YAML from 'js-yaml'; +import { HeuristicsSchema, LanguagesScema, VendorSchema } from '../../types/schema.js'; +import { loadFile, parseGeneratedDataFile } from './loadDataFiles.js'; + +type LoadedData = { + langData: LanguagesScema; + vendorData: VendorSchema; + docData: VendorSchema; + heuristicsData: HeuristicsSchema; + generatedData: string[]; + vendorPaths: string[]; +}; + +let data: LoadedData = null!; + +async function initRetrieveData(offline: boolean): Promise { + // Only load the data on mont + if (data) return; + + const langData = (await loadFile('languages.yml', offline).then(YAML.load)) as LanguagesScema; + const vendorData = (await loadFile('vendor.yml', offline).then(YAML.load)) as VendorSchema; + const docData = (await loadFile('documentation.yml', offline).then(YAML.load)) as VendorSchema; + const heuristicsData = (await loadFile('heuristics.yml', offline).then(YAML.load)) as HeuristicsSchema; + const generatedData = (await loadFile('generated.rb', offline).then(parseGeneratedDataFile)) as string[]; + const vendorPaths = [...vendorData, ...docData, ...generatedData]; + + data = { + langData, + vendorData, + docData, + heuristicsData, + generatedData, + vendorPaths, + }; +} + +/** Load data from github-linguist web repo or cached local file. */ +export default async function retrieveData(offline: boolean): Promise { + await initRetrieveData(offline); + return data; +} diff --git a/src/helpers/norm-path.ts b/src/program/fs/normalisedPath.ts similarity index 90% rename from src/helpers/norm-path.ts rename to src/program/fs/normalisedPath.ts index 1fa9efb..a2eb30a 100644 --- a/src/helpers/norm-path.ts +++ b/src/program/fs/normalisedPath.ts @@ -1,4 +1,4 @@ -import Path from 'path'; +import Path from 'node:path'; export const normPath = function normalisedPath(...inputPaths: string[]) { return Path.join(...inputPaths).replace(/\\/g, '/'); diff --git a/src/helpers/read-file.ts b/src/program/fs/readFile.ts similarity index 95% rename from src/helpers/read-file.ts rename to src/program/fs/readFile.ts index fbd4246..1c3b55b 100644 --- a/src/helpers/read-file.ts +++ b/src/program/fs/readFile.ts @@ -1,4 +1,4 @@ -import FS from 'fs'; +import FS from 'node:fs'; /** * Read part of a file on disc. diff --git a/src/helpers/walk-tree.ts b/src/program/fs/walkTree.ts similarity index 77% rename from src/helpers/walk-tree.ts rename to src/program/fs/walkTree.ts index 2f2d305..d5df394 100644 --- a/src/helpers/walk-tree.ts +++ b/src/program/fs/walkTree.ts @@ -1,32 +1,32 @@ -import FS from 'fs'; -import Path from 'path'; import { Ignore } from 'ignore'; -import parseGitignore from './parse-gitignore'; -import { normPath, normAbsPath } from './norm-path'; +import FS from 'node:fs'; +import Path from 'node:path'; +import parseGitignore from '../parsing/parseGitignore.js'; +import { normAbsPath, normPath } from './normalisedPath.js'; let allFiles: Set; let allFolders: Set; interface WalkInput { /** Whether this is walking the tree from the root */ - init: boolean, + init: boolean; /** The common root absolute path of all folders being checked */ - commonRoot: string, + commonRoot: string; /** The absolute path that each folder is relative to */ - folderRoots: string[], + folderRoots: string[]; /** The absolute path of folders being checked */ - folders: string[], + folders: string[]; /** An instantiated Ignore object listing ignored files */ - ignored: Ignore, -}; + ignored: Ignore; +} interface WalkOutput { - files: string[], - folders: string[], -}; + files: string[]; + folders: string[]; +} /** Generate list of files in a directory. */ -export default function walk(data: WalkInput): WalkOutput { +export default function walkTree(data: WalkInput): WalkOutput { const { init, commonRoot, folderRoots, folders, ignored } = data; // Initialise files and folders lists @@ -41,7 +41,7 @@ export default function walk(data: WalkInput): WalkOutput { const localRoot = folderRoots[0].replace(commonRoot, '').replace(/^\//, ''); // Get list of files and folders inside this folder - const files = FS.readdirSync(folder).map(file => { + const files = FS.readdirSync(folder).map((file) => { // Create path relative to root const base = normAbsPath(folder, file).replace(commonRoot, '.'); // Add trailing slash to mark directories @@ -54,7 +54,7 @@ export default function walk(data: WalkInput): WalkOutput { if (FS.existsSync(gitignoreFilename)) { const gitignoreContents = FS.readFileSync(gitignoreFilename, 'utf-8'); const ignoredPaths = parseGitignore(gitignoreContents); - const rootRelIgnoredPaths = ignoredPaths.map(ignorePath => + const rootRelIgnoredPaths = ignoredPaths.map((ignorePath) => // get absolute path of the ignore glob normPath(folder, ignorePath) // convert abs ignore glob to be relative to the root folder @@ -88,9 +88,8 @@ export default function walk(data: WalkInput): WalkOutput { if (file.endsWith('/')) { // Recurse into subfolders allFolders.add(path); - walk({ init: false, commonRoot, folderRoots, folders: [path], ignored }); - } - else { + walkTree({ init: false, commonRoot, folderRoots, folders: [path], ignored }); + } else { // Add file path to list allFiles.add(path); } @@ -99,12 +98,12 @@ export default function walk(data: WalkInput): WalkOutput { // Recurse into all folders else { for (const i in folders) { - walk({ init: false, commonRoot, folderRoots: [folderRoots[i]], folders: [folders[i]], ignored }); + walkTree({ init: false, commonRoot, folderRoots: [folderRoots[i]], folders: [folders[i]], ignored }); } } // Return absolute files and folders lists return { - files: [...allFiles].map(file => file.replace(/^\./, commonRoot)), + files: [...allFiles].map((file) => file.replace(/^\./, commonRoot)), folders: [...allFolders], }; } diff --git a/src/helpers/parse-gitattributes.ts b/src/program/parsing/parseGItattributes.ts similarity index 89% rename from src/helpers/parse-gitattributes.ts rename to src/program/parsing/parseGItattributes.ts index d243daf..138c334 100644 --- a/src/helpers/parse-gitattributes.ts +++ b/src/program/parsing/parseGItattributes.ts @@ -1,5 +1,5 @@ -import * as T from '../types'; -import { normPath } from './norm-path'; +import * as T from '../../types/types.js'; +import { normPath } from '../fs/normalisedPath.js'; export type FlagAttributes = { 'vendored': boolean | null, @@ -18,7 +18,7 @@ export type ParsedGitattributes = Array<{ /** * Parses a gitattributes file. */ -export default function parseAttributes(content: string, folderRoot: string = '.'): ParsedGitattributes { +export default function parseGitattributes(content: string, folderRoot: string = '.'): ParsedGitattributes { const output: ParsedGitattributes = []; for (const rawLine of content.split('\n')) { diff --git a/src/helpers/parse-gitignore.ts b/src/program/parsing/parseGitignore.ts similarity index 100% rename from src/helpers/parse-gitignore.ts rename to src/program/parsing/parseGitignore.ts diff --git a/src/helpers/convert-pcre.ts b/src/program/utils/pcre.ts similarity index 100% rename from src/helpers/convert-pcre.ts rename to src/program/utils/pcre.ts diff --git a/src/schema.ts b/src/types/schema.ts similarity index 94% rename from src/schema.ts rename to src/types/schema.ts index 8617ebb..76781ea 100644 --- a/src/schema.ts +++ b/src/types/schema.ts @@ -1,4 +1,4 @@ -import { Category, Language } from './types' +import { Category, Language } from './types.js' export interface LanguagesScema { [name: string]: { diff --git a/src/types.ts b/src/types/types.ts similarity index 76% rename from src/types.ts rename to src/types/types.ts index 077e9d2..3a90569 100644 --- a/src/types.ts +++ b/src/types/types.ts @@ -11,7 +11,6 @@ export type AbsFolder = string & {} export type FileGlob = string & {} export interface Options { - fileContent?: string | string[] ignoredFiles?: string[] ignoredLanguages?: Language[] categories?: Category[] @@ -30,15 +29,16 @@ export interface Options { checkModeline?: boolean } +type LinesOfCode = { + total: Integer + content: Integer +} + export interface Results { files: { count: Integer bytes: Bytes - lines: { - total: Integer - content: Integer - code: Integer - } + lines: LinesOfCode /** Note: Results use slashes as delimiters even on Windows. */ results: Record alternatives: Record @@ -46,32 +46,23 @@ export interface Results { languages: { count: Integer bytes: Bytes - lines: { - total: Integer - content: Integer - code: Integer - } + lines: LinesOfCode results: Record } unknown: { count: Integer bytes: Bytes - lines: { - total: Integer - content: Integer - code: Integer - } + lines: LinesOfCode extensions: Record filenames: Record } + repository: Record } diff --git a/test/expected.json b/test/expected.json index 90c7971..27c7604 100644 --- a/test/expected.json +++ b/test/expected.json @@ -2,7 +2,7 @@ "files": { "count": 12, "bytes": 199, - "lines": { "total": 27, "content": 16, "code": 11 }, + "lines": { "total": 27, "content": 16 }, "results": { "~/al.al": "Perl", "~/alternatives.asc": "AGS Script", @@ -18,27 +18,74 @@ "~/unknown": null }, "alternatives": { - "~/alternatives.asc": [ "AsciiDoc", "Public Key" ] + "~/alternatives.asc": ["AsciiDoc", "Public Key"] } }, "languages": { "count": 8, "bytes": 190, "results": { - "Perl": { "type": "programming", "bytes": 0, "lines": { "total": 1, "content": 0, "code": 0 },"color": "#0298c3" }, - "AGS Script": { "type": "programming", "bytes": 14, "lines": { "total": 2, "content": 1, "code": 1 },"color": "#B9D9FF" }, - "JSON": { "type": "data", "bytes": 8, "lines": { "total": 4, "content": 2, "code": 2 },"color": "#292929"}, - "JavaScript": { "type": "programming", "bytes": 23, "lines": { "total": 4, "content": 3, "code": 3 },"color": "#f1e05a" }, - "Text": { "type": "prose", "bytes": 0, "lines": { "total": 1, "content": 0, "code": 0 } }, - "C": { "type": "programming", "bytes": 130, "lines": { "total": 10, "content": 8, "code": 4 }, "color": "#555555"}, - "C++": { "type": "programming", "bytes": 15, "lines": { "total": 2, "content": 1, "code": 0 }, "color": "#f34b7d" }, - "TOML": { "type": "data", "bytes": 0, "lines": { "total": 1, "content": 0, "code": 0 }, "color": "#9c4221" } + "Perl": { + "count": 1, + "type": "programming", + "bytes": 0, + "lines": { "total": 1, "content": 0 }, + "color": "#0298c3" + }, + "AGS Script": { + "count": 1, + "type": "programming", + "bytes": 14, + "lines": { "total": 2, "content": 1 }, + "color": "#B9D9FF" + }, + "JSON": { + "count": 2, + "type": "data", + "bytes": 8, + "lines": { "total": 4, "content": 2 }, + "color": "#292929" + }, + "JavaScript": { + "count": 3, + "type": "programming", + "bytes": 23, + "lines": { "total": 4, "content": 3 }, + "color": "#f1e05a" + }, + "Text": { + "count": 1, + "type": "prose", + "bytes": 0, + "lines": { "total": 1, "content": 0 } + }, + "C": { + "count": 1, + "type": "programming", + "bytes": 130, + "lines": { "total": 10, "content": 8 }, + "color": "#555555" + }, + "C++": { + "count": 1, + "type": "programming", + "bytes": 15, + "lines": { "total": 2, "content": 1 }, + "color": "#f34b7d" + }, + "TOML": { + "count": 1, + "type": "data", + "bytes": 0, + "lines": { "total": 1, "content": 0 }, + "color": "#9c4221" + } } }, "unknown": { "count": 1, "bytes": 9, - "lines": { "total": 2, "content": 1, "code": 1 }, + "lines": { "total": 2, "content": 1 }, "extensions": {}, "filenames": { "unknown": 9 diff --git a/test/folder.js b/test/folder.js index 349c9f2..da19c71 100644 --- a/test/folder.js +++ b/test/folder.js @@ -1,16 +1,18 @@ -const fs = require('fs'); -const linguist = require('..'); -const { updatedDiff } = require('deep-object-diff'); +import { updatedDiff } from 'deep-object-diff'; +import FS from 'node:fs'; +import { dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import linguist from '../dist/index.js'; async function testFolder() { - console.info('-'.repeat(11) + '\nFolder test\n' + '-'.repeat(11)); - const samplesFolder = __dirname.replace(/\\/g, '/') + '/samples'; - const expectedJson = fs.readFileSync(__dirname + '/expected.json', { encoding: 'utf8' }); + console.info('-'.repeat(11) + ' Folder test ' + '-'.repeat(11)); + const curFolder = dirname(fileURLToPath(import.meta.url)); + const samplesFolder = curFolder.replace(/\\/g, '/') + '/samples'; + const expectedJson = FS.readFileSync(curFolder + '/expected.json', { encoding: 'utf8' }); const expected = JSON.parse(expectedJson.replace(/~/g, samplesFolder)); const actual = await linguist(samplesFolder); const diff = updatedDiff(expected, actual); - console.dir(actual, { depth: null }); if (JSON.stringify(diff) === '{}') { console.info('Results match expected'); } diff --git a/test/perf.js b/test/perf.js index 5d8d134..b75ff20 100644 --- a/test/perf.js +++ b/test/perf.js @@ -1,4 +1,4 @@ -const linguist = require('..'); +import linguist from '../dist/index.js'; async function perfTest() { let time = 0; diff --git a/test/unit.js b/test/unit.js index 0264e1b..aa905a5 100644 --- a/test/unit.js +++ b/test/unit.js @@ -1,14 +1,14 @@ -const linguist = require('..'); +import linguist from '../dist/index.js'; let i = 0; let errors = 0; function desc(text) { - console.info(`Testing: ${text}`); + console.info(` Testing: ${text}`); } async function test([filename, fileContent = ''], [type, testVal]) { - const actual = await linguist(filename, { fileContent, childLanguages: true }); + const actual = await linguist({ [filename]: fileContent }, { childLanguages: true }); const testContent = { 'files': actual.files.results[filename], 'size': actual.files.bytes, @@ -17,17 +17,14 @@ async function test([filename, fileContent = ''], [type, testVal]) { }[type]; const result = testContent === testVal; i = `${+i + 1}`.padStart(2, '0'); - if (result) { - console.info(`- #${i} passed: '${filename}' is ${testVal}`); - } - else { + if (!result) { errors++; console.error(`! #${i} failed: '${filename}' is ${testContent} instead of ${testVal}`); } } async function unitTest() { - console.info('-'.repeat(10) + '\nUnit tests\n' + '-'.repeat(10)); + console.info('-'.repeat(10) + ' Unit tests ' + '-'.repeat(10)); desc('metadata'); await test(['file_size', '0123456789'], ['size', 10]); await test(['empty', ''], ['size', 0]); diff --git a/tsconfig.json b/tsconfig.json index 5dbba49..91264ce 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,9 +9,9 @@ /* Examples: https://github.com/tsconfig/bases */ /* Basic Options */ - "target": "es2019", // Node 12 - "module": "commonjs", - "lib": ["es2020"], + "target": "ES2023", + "module": "NodeNext", + "lib": ["ESNext"], //"allowJs": true, //"checkJs": true, //"jsx": "preserve", @@ -46,7 +46,7 @@ //"noPropertyAccessFromIndexSignature": true, /* Module Resolution Options */ - "moduleResolution": "node", + "moduleResolution": "nodenext", "resolveJsonModule": true, //"baseUrl": "./", //"paths": {},