AG-36016 Fix build size for CSS Tokenizer and AGTree

Merge in ADGUARD-FILTERS/tsurlfilter from fix/AG-36016-1 to master Squashed commit of the following: commit ad53c10 Merge: 6e2bb9e ff117a6 Author: scripthunter7 <[email protected]> Date: Thu Sep 19 13:32:55 2024 +0200 Merge branch 'master' into fix/AG-36016-1 commit 6e2bb9e Author: scripthunter7 <[email protected]> Date: Thu Sep 19 13:20:00 2024 +0200 change operator commit 8cc6f29 Author: scripthunter7 <[email protected]> Date: Thu Sep 19 10:11:53 2024 +0200 bump version and changelog for css tokenizer commit da3ddaf Author: scripthunter7 <[email protected]> Date: Thu Sep 19 10:11:43 2024 +0200 bump version and changelog for agtree commit 026a507 Author: scripthunter7 <[email protected]> Date: Thu Sep 19 10:10:00 2024 +0200 smoke tests for css tokenizer commit 715e1b3 Author: scripthunter7 <[email protected]> Date: Thu Sep 19 10:00:10 2024 +0200 small fix for css tokenizer build commit b3a2a14 Author: scripthunter7 <[email protected]> Date: Thu Sep 19 09:59:06 2024 +0200 optimize compatibility tables build for agtree
AdguardTeam · Sep 19, 2024 · cf96c17 · cf96c17
1 parent ff117a6
commit cf96c17
Show file tree

Hide file tree

Showing 23 changed files with 273 additions and 18 deletions.
diff --git a/packages/agtree/CHANGELOG.md b/packages/agtree/CHANGELOG.md
@@ -8,6 +8,14 @@ The format is based on [Keep a Changelog][keepachangelog], and this project adhe
 [keepachangelog]: https://keepachangelog.com/en/1.0.0/
 [semver]: https://semver.org/spec/v2.0.0.html
 
+## [2.1.1] - 2024-09-19
+
+### Fixed
+
+- Optimized build size.
+
+[2.1.1]: https://github.com/AdguardTeam/tsurlfilter/releases/tag/agtree-v2.1.1
+
 ## [2.1.0] - 2024-09-18
 
 ### Added

diff --git a/packages/agtree/package.json b/packages/agtree/package.json
@@ -1,6 +1,6 @@
 {
     "name": "@adguard/agtree",
-    "version": "2.1.0",
+    "version": "2.1.1",
     "description": "Tool set for working with adblock filter lists",
     "keywords": [
         "adblock",
@@ -37,13 +37,16 @@
         }
     },
     "files": [
-        "dist"
+        "dist/agtree.d.ts",
+        "dist/agtree.js",
+        "dist/agtree.mjs",
+        "dist/compatibility-table-data.js"
     ],
     "engines": {
         "node": ">=18"
     },
     "scripts": {
-        "build": "pnpm clean && pnpm build-txt && pnpm build-compatibility-tables && pnpm build-compatibility-tables-wiki && pnpm rollup --config rollup.config.ts --configPlugin typescript && rimraf dist/types",
+        "build": "pnpm clean && pnpm build-txt && pnpm build-compatibility-tables && pnpm build-compatibility-tables-wiki && pnpm rollup --config rollup.config.ts --configPlugin typescript && rimraf dist/types && rimraf dist/compatibility-table-data.json",
         "build-txt": "tsx scripts/build-txt.ts",
         "build-compatibility-tables": "tsx scripts/build-compatibility-tables.ts",
         "build-compatibility-tables-wiki": "tsx scripts/build-compatibility-tables-wiki.ts",
@@ -63,6 +66,7 @@
         "@rollup/plugin-commonjs": "^25.0.2",
         "@rollup/plugin-json": "^6.0.0",
         "@rollup/plugin-node-resolve": "^15.1.0",
+        "@rollup/plugin-replace": "^5.0.7",
         "@rollup/plugin-swc": "^0.4.0",
         "@rollup/plugin-typescript": "^11.1.2",
         "@swc/core": "^1.7.24",

diff --git a/packages/agtree/rollup.config.ts b/packages/agtree/rollup.config.ts
@@ -10,6 +10,7 @@ import externals from 'rollup-plugin-node-externals';
 import dtsPlugin from 'rollup-plugin-dts';
 import alias from '@rollup/plugin-alias';
 import json from '@rollup/plugin-json';
+import replace from '@rollup/plugin-replace';
 import path from 'node:path';
 import { readFileSync } from 'node:fs';
 
@@ -20,6 +21,7 @@ const ROOT_DIR = __dirname;
 const BASE_FILE_NAME = 'agtree';
 const BASE_NAME = 'AGTree';
 const PKG_FILE_NAME = 'package.json';
+const COMPATIBILITY_TABLE_FILENAME = 'compatibility-table-data.js';
 
 const distDir = path.join(ROOT_DIR, 'dist');
 const pkgFileLocation = path.join(ROOT_DIR, PKG_FILE_NAME);
@@ -52,13 +54,12 @@ const banner = `/*
 
 // Common plugins for all types of builds
 export const commonPlugins = [
-    json({ preferConst: true }),
     alias({
         entries: [
             // Replace dynamic compatibility table data builder with the pre-built data file
             {
                 find: './compatibility-table-data',
-                replacement: path.resolve(ROOT_DIR, 'dist/compatibility-tables.json'),
+                replacement: path.resolve(distDir, COMPATIBILITY_TABLE_FILENAME),
             },
             // Add ".js" extension to all imports of the "semver" package, eg "semver/functions/..."
             // We need this because we import functions from the "semver" package directly,
@@ -73,9 +74,10 @@ export const commonPlugins = [
             },
         ],
     }),
-    commonjs({ sourceMap: false }),
-    resolve({ preferBuiltins: false }),
     externals(),
+    json({ preferConst: true }),
+    resolve({ preferBuiltins: false }),
+    commonjs({ sourceMap: false }),
     typescript({
         tsconfig: path.join(ROOT_DIR, 'tsconfig.json'),
         compilerOptions: {
@@ -85,10 +87,41 @@ export const commonPlugins = [
         },
         include: [path.join(ROOT_DIR, './src/**/*.ts')],
         exclude: [path.join(ROOT_DIR, './node_modules'), path.join(ROOT_DIR, './test')],
-        outputToFilesystem: true,
+        outputToFilesystem: false,
+    }),
+    replace({
+        preventAssignment: true,
+        delimiters: ["'", "'"],
+        values: {
+            [path.join(distDir, COMPATIBILITY_TABLE_FILENAME)]: `'./${COMPATIBILITY_TABLE_FILENAME}'`,
+        },
     }),
 ];
 
+const compatibilityTablesBanner = `/**
+* @file Compatibility tables data for AGTree
+*
+* This file is auto-generated from YAML files in the "compatibility-tables" directory.
+* It is optimized for better runtime usage and storage efficiency.
+*
+* We use "shared" section to share the same values between different map keys
+* to reduce the storage usage.
+*/`;
+
+const compatibilityTables = {
+    input: path.join(distDir, 'compatibility-table-data.json'),
+    output: [
+        {
+            file: path.join(distDir, COMPATIBILITY_TABLE_FILENAME),
+            format: 'cjs',
+            exports: 'named',
+            sourcemap: false,
+            banner: compatibilityTablesBanner,
+        },
+    ],
+    plugins: [json()],
+};
+
 const node = {
     input: path.join(ROOT_DIR, 'src/index.ts'),
     output: [
@@ -106,6 +139,9 @@ const node = {
             banner,
         },
     ],
+    external: [
+        path.resolve(distDir, COMPATIBILITY_TABLE_FILENAME),
+    ],
     plugins: commonPlugins,
 };
 
@@ -127,4 +163,4 @@ const dts = {
 };
 
 // Export build configs for Rollup
-export default [node, dts];
+export default [compatibilityTables, node, dts];
diff --git a/packages/agtree/scripts/build-compatibility-tables.ts b/packages/agtree/scripts/build-compatibility-tables.ts
@@ -11,7 +11,7 @@ import path from 'path';
 import * as data from '../src/compatibility-tables/compatibility-table-data';
 
 const DIST_FOLDER_NAME = 'dist';
-const OUTPUT_FILE_NAME = 'compatibility-tables.json';
+const OUTPUT_FILE_NAME = 'compatibility-table-data.json';
 
 /**
  * Main function.

diff --git a/packages/agtree/test/smoke/cjs/index.js b/packages/agtree/test/smoke/cjs/index.js
@@ -1,8 +1,12 @@
-const { RuleParser } = require('@adguard/agtree');
+const { RuleParser, modifiersCompatibilityTable, SpecificPlatform } = require('@adguard/agtree');
 const { ok } = require('assert');
 
 const ruleNode = RuleParser.parse('||example.com^');
 
 ok(ruleNode);
 
+const modifierData = modifiersCompatibilityTable.getSingle('third-party', SpecificPlatform.AdgExtChrome);
+
+ok(modifierData);
+
 console.log('Smoke test passed');
diff --git a/packages/agtree/test/smoke/esm/index.js b/packages/agtree/test/smoke/esm/index.js
@@ -1,8 +1,12 @@
-import { RuleParser } from '@adguard/agtree';
+import { RuleParser, modifiersCompatibilityTable, SpecificPlatform } from '@adguard/agtree';
 import { ok } from 'assert';
 
 const ruleNode = RuleParser.parse('||example.com^');
 
 ok(ruleNode);
 
+const modifierData = modifiersCompatibilityTable.getSingle('third-party', SpecificPlatform.AdgExtChrome);
+
+ok(modifierData);
+
 console.log('Smoke test passed');
diff --git a/packages/agtree/test/smoke/typescript/index.ts b/packages/agtree/test/smoke/typescript/index.ts
@@ -1,8 +1,12 @@
-import { RuleParser } from '@adguard/agtree';
+import { RuleParser, modifiersCompatibilityTable, SpecificPlatform } from '@adguard/agtree';
 import { ok } from 'assert';
 
 const ruleNode = RuleParser.parse('||example.com^');
 
 ok(ruleNode);
 
+const modifierData = modifiersCompatibilityTable.getSingle('third-party', SpecificPlatform.AdgExtChrome);
+
+ok(modifierData);
+
 console.log('Smoke test passed');
diff --git a/packages/css-tokenizer/.eslintignore b/packages/css-tokenizer/.eslintignore
@@ -1,2 +1,4 @@
 node_modules
 dist
+coverage
+test/smoke
diff --git a/packages/css-tokenizer/CHANGELOG.md b/packages/css-tokenizer/CHANGELOG.md
@@ -7,6 +7,14 @@ The format is based on [Keep a Changelog][keepachangelog], and this project adhe
 [keepachangelog]: https://keepachangelog.com/en/1.0.0/
 [semver]: https://semver.org/spec/v2.0.0.html
 
+## [1.1.1] - 2024-09-19
+
+### Fixed
+
+- Optimized build size.
+
+[1.1.1]: https://github.com/AdguardTeam/tsurlfilter/releases/tag/css-tokenizer-v1.1.1
+
 ## [1.1.0] - 2024-09-18
 
 ### Removed

diff --git a/packages/css-tokenizer/package.json b/packages/css-tokenizer/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@adguard/css-tokenizer",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "description": "CSS / Extended CSS tokenizer",
   "keywords": [
     "css",
@@ -28,7 +28,9 @@
     }
   },
   "files": [
-    "dist"
+    "dist/csstokenizer.d.ts",
+    "dist/csstokenizer.js",
+    "dist/csstokenizer.mjs"
   ],
   "scripts": {
     "build": "pnpm clean && pnpm build-txt && pnpm rollup --config rollup.config.ts --configPlugin @rollup/plugin-json --configPlugin @rollup/plugin-typescript && rimraf dist/types",
@@ -41,7 +43,8 @@
     "lint": "pnpm lint:ts && pnpm lint:md",
     "lint:md": "markdownlint .",
     "lint:ts": "eslint . --cache --ext .ts",
-    "test": "jest --runInBand"
+    "test": "jest --runInBand",
+    "test:smoke": "(cd test/smoke/esm && pnpm test) && (cd test/smoke/cjs && pnpm test) && (cd test/smoke/typescript && pnpm test)"
   },
   "devDependencies": {
     "@csstools/css-tokenizer": "^2.2.1",

diff --git a/packages/css-tokenizer/rollup.config.ts b/packages/css-tokenizer/rollup.config.ts
@@ -61,7 +61,7 @@ const commonPlugins = [
         },
         include: ['./src/**/*.ts'],
         exclude: ['./node_modules', './test'],
-        outputToFilesystem: true,
+        outputToFilesystem: false,
     }),
 ];
 

diff --git a/packages/css-tokenizer/test/smoke/cjs/index.js b/packages/css-tokenizer/test/smoke/cjs/index.js
@@ -0,0 +1,12 @@
+const { tokenize } = require('@adguard/css-tokenizer');
+const { ok } = require('assert');
+
+let tokensCount = 0;
+
+tokenize('div { color: red; }', () => {
+    tokensCount += 1;
+});
+
+ok(tokensCount > 0);
+
+console.log('Smoke test passed');
diff --git a/packages/css-tokenizer/test/smoke/cjs/package.json b/packages/css-tokenizer/test/smoke/cjs/package.json
@@ -0,0 +1,11 @@
+{
+  "name": "cjs",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "author": "Adguard Software Ltd.",
+  "scripts": {
+    "start": "node index.js",
+    "test": "./test.sh"
+  }
+}
diff --git a/packages/css-tokenizer/test/smoke/cjs/test.sh b/packages/css-tokenizer/test/smoke/cjs/test.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -e  # Exit on error
+
+# pack @adguard/css-tokenizer
+curr_path="test/smoke/cjs"
+csstokenizer="css-tokenizer.tgz"
+nm_path="node_modules"
+
+# Define cleanup function
+cleanup() {
+    echo "Cleaning up..."
+    rm -f $csstokenizer && rm -rf $nm_path
+    echo "Cleanup complete"
+}
+
+# Set trap to execute the cleanup function on script exit
+trap cleanup EXIT
+
+(cd ../../.. && pnpm pack && mv adguard-css-tokenizer-*.tgz "$curr_path/$csstokenizer")
+
+# unzip to @adguard/tsurlfilter to node_modules
+csstokenizer_node_modules=$nm_path"/@adguard/css-tokenizer"
+mkdir -p $csstokenizer_node_modules
+tar -xzf $csstokenizer --strip-components=1 -C $csstokenizer_node_modules
+
+pnpm start
+echo "Test successfully built."
diff --git a/packages/css-tokenizer/test/smoke/esm/index.js b/packages/css-tokenizer/test/smoke/esm/index.js
@@ -0,0 +1,12 @@
+import { tokenize } from '@adguard/css-tokenizer';
+import { ok } from 'assert';
+
+let tokensCount = 0;
+
+tokenize('div { color: red; }', () => {
+    tokensCount += 1;
+});
+
+ok(tokensCount > 0);
+
+console.log('Smoke test passed');
diff --git a/packages/css-tokenizer/test/smoke/esm/package.json b/packages/css-tokenizer/test/smoke/esm/package.json
@@ -0,0 +1,12 @@
+{
+  "name": "esm",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "author": "Adguard Software Ltd.",
+  "type": "module",
+  "scripts": {
+    "start": "node index.js",
+    "test": "./test.sh"
+  }
+}
diff --git a/packages/css-tokenizer/test/smoke/esm/test.sh b/packages/css-tokenizer/test/smoke/esm/test.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -e  # Exit on error
+
+# pack @adguard/css-tokenizer
+curr_path="test/smoke/esm"
+csstokenizer="css-tokenizer.tgz"
+nm_path="node_modules"
+
+# Define cleanup function
+cleanup() {
+    echo "Cleaning up..."
+    rm -f $csstokenizer && rm -rf $nm_path
+    echo "Cleanup complete"
+}
+
+# Set trap to execute the cleanup function on script exit
+trap cleanup EXIT
+
+(cd ../../.. && pnpm pack && mv adguard-css-tokenizer-*.tgz "$curr_path/$csstokenizer")
+
+# unzip to @adguard/tsurlfilter to node_modules
+csstokenizer_node_modules=$nm_path"/@adguard/css-tokenizer"
+mkdir -p $csstokenizer_node_modules
+tar -xzf $csstokenizer --strip-components=1 -C $csstokenizer_node_modules
+
+pnpm start
+echo "Test successfully built."
diff --git a/packages/css-tokenizer/test/smoke/typescript/index.ts b/packages/css-tokenizer/test/smoke/typescript/index.ts
@@ -0,0 +1,12 @@
+import { tokenize } from '@adguard/css-tokenizer';
+import { ok } from 'assert';
+
+let tokensCount = 0;
+
+tokenize('div { color: red; }', () => {
+    tokensCount += 1;
+});
+
+ok(tokensCount > 0);
+
+console.log('Smoke test passed');
diff --git a/packages/css-tokenizer/test/smoke/typescript/package.json b/packages/css-tokenizer/test/smoke/typescript/package.json
@@ -0,0 +1,11 @@
+{
+  "name": "typescript",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "author": "Adguard Software Ltd.",
+  "scripts": {
+    "start": "tsc --noEmit",
+    "test": "./test.sh"
+  }
+}