Complete new bench

This commit is contained in:
Wilson Lin 2021-08-08 23:11:05 +10:00
parent 21297d053a
commit 38f186f73e
25 changed files with 204 additions and 351 deletions

View File

@ -47,9 +47,7 @@ jobs:
- name: Build bench
working-directory: ./bench
run: |
sudo apt install -y build-essential
npm install
./build.sh
./build
- name: Set up Backblaze B2 CLI
uses: wilsonzlin/setup-b2@v3
@ -58,7 +56,7 @@ jobs:
working-directory: ./bench
run: |
b2 authorize-account ${{ secrets.CICD_CLI_B2_KEY_ID }} ${{ secrets.CICD_CLI_B2_APPLICATION_KEY }}
./bench.sh
b2 sync ./results/ b2://${{ secrets.CICD_CLI_B2_BUCKET_NAME }}/minify-html/bench/${{ steps.version.outputs.VERSION }}/js/
HTML_ONLY=1 ./bench.sh
b2 sync ./results/ b2://${{ secrets.CICD_CLI_B2_BUCKET_NAME }}/minify-html/bench/${{ steps.version.outputs.VERSION }}/core/
./run
b2 sync ./graphs/ b2://${{ secrets.CICD_CLI_B2_BUCKET_NAME }}/minify-html/bench/${{ steps.version.outputs.VERSION }}/js/
MHB_HTML_ONLY=1 ./run
b2 sync ./graphs/ b2://${{ secrets.CICD_CLI_B2_BUCKET_NAME }}/minify-html/bench/${{ steps.version.outputs.VERSION }}/core/

View File

@ -19,7 +19,8 @@ A Rust HTML minifier meticulously optimised for speed and effectiveness, with bi
Comparison with [html-minfier](https://github.com/kangax/html-minifier) and [minimize](https://github.com/Swaagie/minimize), run on the top web pages. [See the breakdown here.](./bench)
<img alt="Chart showing speed and compression of HTML minifiers" src="https://wilsonl.in/minify-html/bench/0.6.0/core/average-combined.png">
<img alt="Chart showing speed of HTML minifiers" src="https://wilsonl.in/minify-html/bench/0.6.0/core/average-speeds.png">
<img alt="Chart showing compression of HTML minifiers" src="https://wilsonl.in/minify-html/bench/0.6.0/core/average-sizes.png">
Need even faster performance? Check the [one](https://github.com/wilsonzlin/minify-html/tree/one) branch.

1
bench/.gitignore vendored
View File

@ -1 +1,2 @@
/graphs/
/results/

View File

@ -1,17 +1,25 @@
const results = require("./results");
const https = require("https");
const path = require("path");
const fs = require("fs/promises");
const colours = {
"minify-html": "#041f60",
"@minify-html/js": "#1f77b4",
minimize: "#ff7f0e",
"html-minifier": "#2ca02c",
const GRAPHS_DIR = path.join(__dirname, "graphs");
const SPEEDS_GRAPH = path.join(GRAPHS_DIR, "speeds.png");
const SIZES_GRAPH = path.join(GRAPHS_DIR, "sizes.png");
const AVERAGE_SPEEDS_GRAPH = path.join(GRAPHS_DIR, "average-speeds.png");
const AVERAGE_SIZES_GRAPH = path.join(GRAPHS_DIR, "average-sizes.png");
const speedColours = {
"@minify-html/js": "#2e61bd",
"minify-html": "#2e61bd",
"minify-html-onepass": "#222",
};
const defaultSpeedColour = "rgb(188, 188, 188)";
const COLOUR_SPEED_PRIMARY = "#2e61bd";
const COLOUR_SPEED_SECONDARY = "rgb(188, 188, 188)";
const COLOUR_SIZE_PRIMARY = "#64acce";
const COLOUR_SIZE_SECONDARY = "rgb(224, 224, 224)";
const sizeColours = {
"minify-html": "#2e61bd",
};
const defaultSizeColour = "rgb(188, 188, 188)";
const breakdownChartOptions = (title) => ({
options: {
@ -30,20 +38,21 @@ const breakdownChartOptions = (title) => ({
scales: {
xAxes: [
{
barPercentage: 0.25,
gridLines: {
color: "#e2e2e2",
color: "#f2f2f2",
},
ticks: {
fontColor: "#666",
callback: "$$$_____REPLACE_WITH_TICK_CALLBACK_____$$$",
fontColor: "#999",
fontSize: 20,
},
},
],
yAxes: [
{
barPercentage: 0.5,
gridLines: {
color: "#ccc",
color: "#aaa",
},
ticks: {
fontColor: "#666",
@ -61,10 +70,10 @@ const axisLabel = (fontColor, labelString) => ({
fontSize: 24,
fontStyle: "bold",
labelString,
padding: 16,
padding: 12,
});
const combinedChartOptions = () => ({
const averageChartOptions = (label) => ({
options: {
legend: {
display: false,
@ -77,45 +86,30 @@ const combinedChartOptions = () => ({
},
ticks: {
fontColor: "#555",
fontSize: 24,
fontSize: 16,
},
},
],
yAxes: [
{
id: "y1",
type: "linear",
scaleLabel: axisLabel(COLOUR_SPEED_PRIMARY, "Performance"),
scaleLabel: axisLabel("#222", label),
position: "left",
ticks: {
callback: "$$$_____REPLACE_WITH_TICK_CALLBACK_____$$$",
fontColor: COLOUR_SPEED_PRIMARY,
fontSize: 24,
fontColor: "#222",
fontSize: 16,
},
gridLines: {
color: "#eee",
},
},
{
id: "y2",
type: "linear",
scaleLabel: axisLabel(COLOUR_SIZE_PRIMARY, "Average size reduction"),
position: "right",
ticks: {
callback: "$$$_____REPLACE_WITH_TICK_CALLBACK_____$$$",
fontColor: COLOUR_SIZE_PRIMARY,
fontSize: 24,
},
gridLines: {
display: false,
},
},
],
},
},
});
const renderChart = (cfg) =>
const renderChart = (cfg, width, height) =>
new Promise((resolve, reject) => {
const req = https.request("https://quickchart.io/chart", {
method: "POST",
@ -141,83 +135,109 @@ const renderChart = (cfg) =>
'"$$$_____REPLACE_WITH_TICK_CALLBACK_____$$$"',
"function(value) {return Math.round(value * 10000) / 100 + '%';}"
),
width: 1333,
height: 768,
width,
height,
format: "png",
})
);
});
(async () => {
const averageSpeeds = results
.getSpeedResults()
.getAverageRelativeSpeedPerMinifier("@minify-html/js");
const averageSizes = results
.getSizeResults()
.getAverageRelativeSizePerMinifier();
const averageLabels = ["minimize", "html-minifier", "@minify-html/js"];
await fs.mkdir(GRAPHS_DIR, { recursive: true });
results.writeAverageCombinedGraph(
await renderChart({
type: "bar",
data: {
labels: averageLabels,
datasets: [
{
yAxisID: "y1",
backgroundColor: averageLabels.map((n) =>
n === "@minify-html/js"
? COLOUR_SPEED_PRIMARY
: COLOUR_SPEED_SECONDARY
),
data: averageLabels.map((n) => averageSpeeds.get(n)),
},
{
yAxisID: "y2",
backgroundColor: averageLabels.map((n) =>
n === "@minify-html/js"
? COLOUR_SIZE_PRIMARY
: COLOUR_SIZE_SECONDARY
),
data: averageLabels.map((n) => 1 - averageSizes.get(n)),
},
],
const res = results.calculate();
const speedMinifiers = [...res.minifiers].sort(
(a, b) => res.minifierAvgOps[a] - res.minifierAvgOps[b]
);
const sizeMinifiers = ["minimize", "html-minifier", "minify-html"];
const inputs = Object.keys(res.inputSizes).sort();
await fs.writeFile(
AVERAGE_SPEEDS_GRAPH,
await renderChart(
{
type: "bar",
data: {
labels: speedMinifiers.map(m => m.replace(" (", "\n(")),
datasets: [
{
backgroundColor: speedMinifiers.map(
(n) => speedColours[n] ?? defaultSpeedColour
),
data: speedMinifiers.map(
(m) => res.minifierAvgOps[m] / res.maxMinifierAvgOps
),
},
],
},
...averageChartOptions("Performance"),
},
...combinedChartOptions(),
})
1024,
768
)
);
const speeds = results
.getSpeedResults()
.getRelativeFileSpeedsPerMinifier("@minify-html/js");
results.writeSpeedsGraph(
await renderChart({
type: "bar",
data: {
labels: speeds[0][1].map(([n]) => n),
datasets: speeds.map(([minifier, fileSpeeds]) => ({
label: minifier,
backgroundColor: colours[minifier],
data: fileSpeeds.map(([_, speed]) => speed),
})),
await fs.writeFile(
AVERAGE_SIZES_GRAPH,
await renderChart(
{
type: "bar",
data: {
labels: sizeMinifiers.map(m => m.replace(" (", "\n(")),
datasets: [
{
backgroundColor: sizeMinifiers.map(
(n) => sizeColours[n] ?? defaultSizeColour
),
data: sizeMinifiers.map((m) => res.minifierAvgReduction[m]),
},
],
},
...averageChartOptions("Reduction"),
},
...breakdownChartOptions("Operations per second (higher is better)"),
})
1024,
768
)
);
const sizes = results.getSizeResults().getRelativeFileSizesPerMinifier();
results.writeSizesGraph(
await renderChart({
type: "bar",
data: {
labels: sizes[0][1].map(([n]) => n),
datasets: sizes.map(([minifier, fileSizes]) => ({
label: minifier,
backgroundColor: colours[minifier],
data: fileSizes.map(([_, size]) => size),
})),
await fs.writeFile(
SPEEDS_GRAPH,
await renderChart(
{
type: "horizontalBar",
data: {
labels: inputs,
datasets: speedMinifiers.map((minifier) => ({
label: minifier,
data: inputs.map(
(input) =>
res.perInputOps[minifier][input] / res.maxInputOps[input]
),
})),
},
...breakdownChartOptions("Operations per second (higher is better)"),
},
...breakdownChartOptions("Minified size (lower is better)"),
})
900,
1000
)
);
await fs.writeFile(
SIZES_GRAPH,
await renderChart(
{
type: "horizontalBar",
data: {
labels: inputs,
datasets: sizeMinifiers.map((minifier) => ({
label: minifier,
data: inputs.map((input) => res.perInputReduction[minifier][input]),
})),
},
...breakdownChartOptions("Size reduction (higher is better)"),
},
900,
1000
)
);
})();

View File

@ -1,99 +1,67 @@
const minifiers = require("./minifiers");
const tests = require("./tests");
const { join } = require("path");
const { mkdirSync, readFileSync, writeFileSync } = require("fs");
const fs = require("fs");
const path = require("path");
const RESULTS_DIR = join(__dirname, "results");
const SPEEDS_JSON = join(RESULTS_DIR, "speeds.json");
const SPEEDS_GRAPH = join(RESULTS_DIR, "speeds.png");
const AVERAGE_COMBINED_GRAPH = join(RESULTS_DIR, "average-combined.png");
const AVERAGE_SPEEDS_GRAPH = join(RESULTS_DIR, "average-speeds.png");
const SIZES_JSON = join(RESULTS_DIR, "sizes.json");
const SIZES_GRAPH = join(RESULTS_DIR, "sizes.png");
const AVERAGE_SIZES_GRAPH = join(RESULTS_DIR, "average-sizes.png");
const minifierNames = Object.keys(minifiers);
const testNames = tests.map((t) => t.name);
mkdirSync(RESULTS_DIR, { recursive: true });
const RESULTS_DIR = path.join(__dirname, "results");
const INPUTS_DIR = path.join(__dirname, "inputs");
module.exports = {
writeSpeedResults(speeds) {
writeFileSync(SPEEDS_JSON, JSON.stringify(speeds, null, 2));
},
writeSizeResults(sizes) {
writeFileSync(SIZES_JSON, JSON.stringify(sizes, null, 2));
},
writeAverageCombinedGraph(data) {
writeFileSync(AVERAGE_COMBINED_GRAPH, data);
},
writeAverageSpeedsGraph(data) {
writeFileSync(AVERAGE_SPEEDS_GRAPH, data);
},
writeSpeedsGraph(data) {
writeFileSync(SPEEDS_GRAPH, data);
},
writeAverageSizesGraph(data) {
writeFileSync(AVERAGE_SIZES_GRAPH, data);
},
writeSizesGraph(data) {
writeFileSync(SIZES_GRAPH, data);
},
getSpeedResults() {
const data = JSON.parse(readFileSync(SPEEDS_JSON, "utf8"));
calculate: () => {
// minifier => avg(ops).
const minifierAvgOps = {};
// minifier => avg(1 - output / original).
const minifierAvgReduction = {};
let maxMinifierAvgOps = 0;
// minifier => input => ops.
const perInputOps = {};
// minifier => input => (1 - output / original).
const perInputReduction = {};
// input => max(ops).
const maxInputOps = {};
const inputSizes = Object.fromEntries(
fs.readdirSync(INPUTS_DIR).map((f) => {
const name = path.basename(f, ".json");
const stats = fs.statSync(path.join(INPUTS_DIR, f));
return [name, stats.size];
})
);
for (const f of fs.readdirSync(RESULTS_DIR)) {
const minifier = decodeURIComponent(path.basename(f, ".json"));
const data = JSON.parse(
fs.readFileSync(path.join(RESULTS_DIR, f), "utf8")
);
for (const [input, size, iterations, seconds] of data) {
const originalSize = inputSizes[input];
const ops = 1 / (seconds / iterations);
const reduction = 1 - size / originalSize;
(minifierAvgOps[minifier] ??= []).push(ops);
(minifierAvgReduction[minifier] ??= []).push(reduction);
(perInputOps[minifier] ??= {})[input] = ops;
(perInputReduction[minifier] ??= {})[input] = reduction;
maxInputOps[input] = Math.max(maxInputOps[input] ?? 0, ops);
}
}
const minifiers = Object.keys(minifierAvgOps);
for (const m of minifiers) {
minifierAvgOps[m] =
minifierAvgOps[m].reduce((sum, ops) => sum + ops, 0) /
minifierAvgOps[m].length;
maxMinifierAvgOps = Math.max(maxMinifierAvgOps, minifierAvgOps[m]);
minifierAvgReduction[m] =
minifierAvgReduction[m].reduce((sum, ops) => sum + ops, 0) /
minifierAvgReduction[m].length;
}
return {
// Get minifier-speed pairs.
getAverageRelativeSpeedPerMinifier(baselineMinifier) {
return new Map(
minifierNames.map((minifier) => [
minifier,
testNames
// Get operations per second for each test.
.map(
(test) => data[test][minifier] / data[test][baselineMinifier]
)
// Sum all test operations per second.
.reduce((sum, c) => sum + c) /
// Divide by tests count to get average operations per second.
testNames.length,
])
);
},
// Get minifier-speeds pairs.
getRelativeFileSpeedsPerMinifier(baselineMinifier) {
return minifierNames.map((minifier) => [
minifier,
testNames.map((test) => [
test,
data[test][minifier] / data[test][baselineMinifier],
]),
]);
},
};
},
getSizeResults() {
const data = JSON.parse(readFileSync(SIZES_JSON, "utf8"));
return {
// Get minifier-size pairs.
getAverageRelativeSizePerMinifier() {
return new Map(
minifierNames.map((minifier) => [
minifier,
testNames
.map((test) => data[test][minifier].relative)
.reduce((sum, c) => sum + c) / testNames.length,
])
);
},
// Get minifier-sizes pairs.
getRelativeFileSizesPerMinifier() {
return minifierNames.map((minifier) => [
minifier,
testNames.map((test) => [test, data[test][minifier].relative]),
]);
},
minifierAvgReduction,
minifierAvgOps,
maxMinifierAvgOps,
perInputOps,
perInputReduction,
maxInputOps,
inputSizes,
minifiers,
};
},
};

View File

@ -14,10 +14,11 @@ const minifyHtmlCfg = minifyHtml.createConfiguration({
const results = fs.readdirSync(inputDir).map((name) => {
const src = fs.readFileSync(path.join(inputDir, name));
const start = process.hrtime.bigint();
let len;
for (let i = 0; i < iterations; i++) {
minifyHtml.minify(src, minifyHtmlCfg);
len = minifyHtml.minify(src, minifyHtmlCfg).byteLength;
}
const elapsed = process.hrtime.bigint() - start;
return [name, Number(elapsed) / 1_000_000_000];
return [name, len, iterations, Number(elapsed) / 1_000_000_000];
});
console.log(JSON.stringify(results));

View File

@ -5,5 +5,5 @@
- `MHB_ITERATIONS`: times to run each input.
- `MHB_INPUT_DIR`: path to directory containing inputs. Files should be read from this directory and used as the inputs.
- `MHB_HTML_ONLY`: if set to `1`, `minify_css` and `minify_js` should be disabled.
- The output should be a JSON array of pairs, where each pair represents the input name and execution time in seconds (as a floating point value).
- The output should be a JSON array of tuples, where each tuples contains the input name, output size, iterations, and execution time in seconds (as a floating point value).
- The execution time should be measured using high-precision monotonic system clocks where possible.

View File

@ -48,10 +48,11 @@ const htmlMinifierCfg = {
const results = fs.readdirSync(inputDir).map((name) => {
const src = fs.readFileSync(path.join(inputDir, name), "utf8");
const start = process.hrtime.bigint();
let len;
for (let i = 0; i < iterations; i++) {
htmlMinifier.minify(src, htmlMinifierCfg);
len = htmlMinifier.minify(src, htmlMinifierCfg).length;
}
const elapsed = process.hrtime.bigint() - start;
return [name, Number(elapsed) / 1_000_000_000];
return [name, len, iterations, Number(elapsed) / 1_000_000_000];
});
console.log(JSON.stringify(results));

View File

@ -11,7 +11,7 @@ fn main() {
let tests = fs::read_dir(input_dir).unwrap().map(|d| d.unwrap());
let mut results: Vec<(String, f64)> = Vec::new();
let mut results: Vec<(String, usize, usize, f64)> = Vec::new();
let cfg = Cfg {
minify_css: !html_only,
minify_js: !html_only,
@ -20,12 +20,13 @@ fn main() {
for t in tests {
let source = fs::read(t.path()).unwrap();
let start = Instant::now();
let mut len = 0;
for _ in 0..iterations {
let mut data = source.to_vec();
let _ = in_place(&mut data, &cfg).expect("failed to minify");
len = in_place(&mut data, &cfg).expect("failed to minify");
};
let elapsed = start.elapsed().as_secs_f64();
results.push((t.file_name().into_string().unwrap(), elapsed));
results.push((t.file_name().into_string().unwrap(), len, iterations, elapsed));
};
serde_json::to_writer(stdout(), &results).unwrap();

View File

@ -11,7 +11,7 @@ fn main() {
let tests = fs::read_dir(input_dir).unwrap().map(|d| d.unwrap());
let mut results: Vec<(String, f64)> = Vec::new();
let mut results: Vec<(String, usize, usize, f64)> = Vec::new();
let mut cfg = Cfg::new();
if !html_only {
cfg.minify_css = true;
@ -21,11 +21,12 @@ fn main() {
for t in tests {
let source = fs::read(t.path()).unwrap();
let start = Instant::now();
let mut len = 0;
for _ in 0..iterations {
let _ = minify(&source, &cfg);
len = minify(&source, &cfg).len();
};
let elapsed = start.elapsed().as_secs_f64();
results.push((t.file_name().into_string().unwrap(), elapsed));
results.push((t.file_name().into_string().unwrap(), len, iterations, elapsed));
};
serde_json::to_writer(stdout(), &results).unwrap();

View File

@ -61,10 +61,11 @@ const plugins = htmlOnly ? [] : [jsCssPlugin];
const results = fs.readdirSync(inputDir).map((name) => {
const src = fs.readFileSync(path.join(inputDir, name), "utf8");
const start = process.hrtime.bigint();
let len;
for (let i = 0; i < iterations; i++) {
new minimize({ plugins }).parse(src);
len = new minimize({ plugins }).parse(src).length;
}
const elapsed = process.hrtime.bigint() - start;
return [name, Number(elapsed) / 1_000_000_000];
return [name, len, iterations, Number(elapsed) / 1_000_000_000];
});
console.log(JSON.stringify(results));

View File

@ -1,42 +0,0 @@
const fs = require("fs");
const path = require("path");
const minifiers = require("./minifiers");
const results = require("./results");
const tests = require("./tests");
const sizes = {};
const setSize = (program, test, result) => {
if (!sizes[test]) {
sizes[test] = {
original: {
absolute: tests.find((t) => t.name === test).contentAsString.length,
relative: 1,
},
};
}
const original = sizes[test].original.absolute;
sizes[test][program] = {
absolute: result,
relative: result / original,
};
};
(async () => {
for (const t of tests) {
for (const m of Object.keys(minifiers)) {
try {
const min = await minifiers[m](t.contentAsString, t.contentAsBuffer);
// If `min` is a Buffer, convert to string (interpret as UTF-8) to get canonical length.
setSize(m, t.name, min.toString().length);
const minPath = path.join(__dirname, "min", m, `${t.name}.html`);
fs.mkdirSync(path.dirname(minPath), { recursive: true });
fs.writeFileSync(minPath, min);
} catch (err) {
console.error(`Failed to run ${m} on test ${t.name}:`);
console.error(err);
process.exit(1);
}
}
}
results.writeSizeResults(sizes);
})();

View File

@ -1,98 +0,0 @@
const benchmark = require("benchmark");
const childProcess = require("child_process");
const minimist = require("minimist");
const path = require("path");
const minifiers = require("./minifiers");
const results = require("./results");
const tests = require("./tests");
const args = minimist(process.argv.slice(2));
const shouldRunRust = !!args.rust;
const cmd = (command, ...args) => {
const throwErr = (msg) => {
throw new Error(`${msg}\n ${command} ${args.join(" ")}`);
};
const { status, signal, error, stdout, stderr } = childProcess.spawnSync(
command,
args.map(String),
{
stdio: ["ignore", "pipe", "pipe"],
encoding: "utf8",
}
);
if (error) {
throwErr(error.message);
}
if (signal) {
throwErr(`Command exited with signal ${signal}`);
}
if (status !== 0) {
throwErr(`Command exited with status ${status}`);
}
if (stderr) {
throwErr(`stderr: ${stderr}`);
}
return stdout;
};
const fromEntries = (entries) => {
if (Object.fromEntries) return Object.fromEntries(entries);
const obj = {};
for (const [prop, val] of entries) obj[prop] = val;
return obj;
};
const runTest = (test) =>
new Promise((resolve, reject) => {
// Run JS libraries.
const suite = new benchmark.Suite();
for (const m of Object.keys(minifiers)) {
suite.add(m, {
defer: true,
fn(deferred) {
Promise.resolve(
minifiers[m](test.contentAsString, test.contentAsBuffer)
).then(() => deferred.resolve());
},
});
}
suite
.on("cycle", (event) => console.info(test.name, event.target.toString()))
.on("complete", () =>
resolve(fromEntries(suite.map((b) => [b.name, b.hz])))
)
.on("error", reject)
.run({ async: true });
});
(async () => {
const speeds = fromEntries(tests.map((t) => [t.name, {}]));
// Run Rust library.
if (shouldRunRust) {
for (const [testName, testOps] of JSON.parse(
cmd(
path.join(
__dirname,
"minify-html-bench",
"target",
"release",
"minify-html-bench"
),
"--iterations",
512,
"--tests",
path.join(__dirname, "tests")
)
)) {
Object.assign(speeds[testName], { ["minify-html"]: testOps });
}
}
for (const t of tests) {
Object.assign(speeds[t.name], await runTest(t));
}
results.writeSpeedResults(speeds);
})();