328 lines
9.5 KiB
JavaScript
328 lines
9.5 KiB
JavaScript
// @ts-check
|
|
/**
|
|
* group_similar.js
|
|
*
|
|
* Reads a markdown file containing timestamped screenshot references like:
|
|
* 
|
|
* 
|
|
* ...
|
|
*
|
|
* Consecutive images that are visually similar get collapsed into a
|
|
* <details> block anchored under the first (non-similar) image.
|
|
*
|
|
* Usage:
|
|
* node group_similar.js <input.md> [output.md] [--threshold 0.005]
|
|
*
|
|
* - output.md defaults to input.md (overwrites in place)
|
|
* - --threshold: similarity cutoff 0-1. Images with a diff ratio
|
|
* BELOW this value are considered "similar". Default: 0.005 (0.5%).
|
|
*
|
|
* Dependencies:
|
|
* npm install sharp
|
|
* npm install --save-dev @types/node (for //@ts-check)
|
|
*/
|
|
|
|
import { readFileSync, writeFileSync } from "node:fs";
|
|
import { resolve, dirname } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import matter from "gray-matter";
|
|
// @ts-ignore - sharp ships no bundled .d.ts; types are not needed at runtime
|
|
import sharp from "sharp";
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Types (JSDoc only — kept in sync with every function signature)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* @typedef {{ type: "image", alt: string, src: string, raw: string }} ImageNode
|
|
* @typedef {{ type: "text", raw: string }} TextNode
|
|
* @typedef {ImageNode | TextNode} Line
|
|
* @typedef {{ blanks: TextNode[], image: ImageNode }} Child
|
|
* @typedef {{ type: "group", anchor: ImageNode, children: Child[] }} GroupNode
|
|
* @typedef {Line | GroupNode} OutputNode
|
|
* @typedef {{ comparisons: number, grouped: number }} Stats
|
|
*/
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Image comparison (returns 0 = identical … 1 = completely different)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const COMPARE_WIDTH = 64;
|
|
const COMPARE_HEIGHT = 64;
|
|
|
|
/**
|
|
* @param {string} imgPath
|
|
* @returns {Promise<Buffer>}
|
|
*/
|
|
async function loadSmall(imgPath) {
|
|
return sharp(imgPath)
|
|
.resize(COMPARE_WIDTH, COMPARE_HEIGHT, { fit: "cover" })
|
|
.raw()
|
|
.toBuffer();
|
|
}
|
|
|
|
/**
|
|
* Mean squared error normalised to [0, 1].
|
|
* @param {Buffer} bufA
|
|
* @param {Buffer} bufB
|
|
* @returns {number}
|
|
*/
|
|
function mse(bufA, bufB) {
|
|
if (bufA.length !== bufB.length) return 1;
|
|
let sum = 0;
|
|
for (let i = 0; i < bufA.length; i++) {
|
|
const d = bufA[i] - bufB[i];
|
|
sum += d * d;
|
|
}
|
|
return sum / bufA.length / (255 * 255);
|
|
}
|
|
|
|
/**
|
|
* @param {string} pathA
|
|
* @param {string} pathB
|
|
* @param {number} threshold
|
|
* @returns {Promise<{ similar: boolean, score: number }>}
|
|
*/
|
|
async function compare(pathA, pathB, threshold) {
|
|
const [a, b] = await Promise.all([loadSmall(pathA), loadSmall(pathB)]);
|
|
const score = mse(a, b);
|
|
return { similar: score < threshold, score };
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Markdown parsing
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const IMG_RE = /^!\[([^\]]*)\]\(([^)]+)\)\s*$/;
|
|
|
|
/**
|
|
* @param {string} line
|
|
* @returns {Line}
|
|
*/
|
|
function parseLine(line) {
|
|
const m = line.match(IMG_RE);
|
|
if (m) return { type: "image", alt: m[1], src: m[2], raw: line };
|
|
return { type: "text", raw: line };
|
|
}
|
|
|
|
/**
|
|
* @param {Line} node
|
|
* @returns {boolean}
|
|
*/
|
|
function isBlank(node) {
|
|
return node.type === "text" && node.raw.trim() === "";
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Core grouping logic
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Walk parsed lines and group consecutive similar images.
|
|
* Blank lines between similar images travel with the group; non-blank
|
|
* non-image text always breaks the lookahead.
|
|
*
|
|
* @param {Line[]} lines
|
|
* @param {string} baseDir
|
|
* @param {number} threshold
|
|
* @param {Stats} stats - mutated in place for the caller's summary
|
|
* @returns {Promise<OutputNode[]>}
|
|
*/
|
|
async function groupImages(lines, baseDir, threshold, stats) {
|
|
/** @type {OutputNode[]} */
|
|
const result = [];
|
|
let i = 0;
|
|
|
|
while (i < lines.length) {
|
|
const line = lines[i];
|
|
|
|
if (line.type !== "image") {
|
|
result.push(line);
|
|
i++;
|
|
continue;
|
|
}
|
|
|
|
/** @type {ImageNode} */
|
|
const anchor = line;
|
|
/** @type {Child[]} */
|
|
const children = [];
|
|
let j = i + 1;
|
|
|
|
// lookahead
|
|
while (j < lines.length) {
|
|
/** @type {TextNode[]} */
|
|
const blanks = [];
|
|
while (j < lines.length && lines[j].type === "text") {
|
|
// isBlank already guarantees type === "text"
|
|
blanks.push(/** @type {TextNode} */ (lines[j]));
|
|
j++;
|
|
}
|
|
|
|
if (j >= lines.length || lines[j].type !== "image") {
|
|
j -= blanks.length; // rewind - blanks stay outside
|
|
break;
|
|
}
|
|
|
|
/** @type {ImageNode} */
|
|
const candidate = /** @type {ImageNode} */ (lines[j]);
|
|
const { similar } = await compare(
|
|
resolve(baseDir, anchor.src.replace(/^\//, "public/")),
|
|
resolve(baseDir, candidate.src.replace(/^\//, "public/")),
|
|
threshold,
|
|
);
|
|
stats.comparisons++;
|
|
|
|
if (similar) {
|
|
stats.grouped++;
|
|
children.push({ blanks, image: candidate });
|
|
j++;
|
|
} else {
|
|
j -= blanks.length; // rewind - blanks stay outside
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (children.length > 0) {
|
|
result.push({ type: "group", anchor, children });
|
|
} else {
|
|
result.push(anchor);
|
|
}
|
|
|
|
i = j;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Rendering
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* @param {ImageNode} node
|
|
* @returns {string}
|
|
*/
|
|
function renderImage(node) {
|
|
return ``;
|
|
}
|
|
|
|
/**
|
|
* @param {GroupNode} node
|
|
* @returns {string}
|
|
*/
|
|
function renderGroup(node) {
|
|
const parts = [];
|
|
|
|
parts.push(renderImage(node.anchor));
|
|
|
|
// First child's blanks are the original spacing between anchor and the
|
|
// next image. Emit them so multiple blank lines are preserved.
|
|
// Then guarantee at least one blank line exists before <details>.
|
|
const firstBlanks = node.children[0].blanks;
|
|
if (firstBlanks.length === 0) parts.push("");
|
|
for (const b of firstBlanks) parts.push(b.raw);
|
|
|
|
parts.push("<details>");
|
|
parts.push(`<summary>${node.anchor.alt}</summary>`);
|
|
parts.push(""); // 1 blank line after <summary>
|
|
|
|
// first child: blanks already emitted above
|
|
parts.push(renderImage(node.children[0].image));
|
|
for (let ci = 1; ci < node.children.length; ci++) {
|
|
const { blanks, image } = node.children[ci];
|
|
for (const b of blanks) parts.push(b.raw);
|
|
parts.push(renderImage(image));
|
|
}
|
|
|
|
parts.push("</details>");
|
|
parts.push("<br/>");
|
|
return parts.join("\n");
|
|
}
|
|
|
|
/**
|
|
* @param {OutputNode[]} nodes
|
|
* @returns {string}
|
|
*/
|
|
function render(nodes) {
|
|
return nodes
|
|
.map((n) => (n.type === "group" ? renderGroup(n) : n.raw))
|
|
.join("\n");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Public API
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Read a markdown file, group similar consecutive images into <details>
|
|
* blocks, and write the result.
|
|
*
|
|
* @param {string} inputPath - path to the source .md file
|
|
* @param {string} outputPath - path to write (may be the same as inputPath)
|
|
* @param {number} [threshold=0.005] - MSE cutoff; lower = stricter grouping
|
|
* @returns {Promise<Stats>} - comparison stats (useful for callers)
|
|
*/
|
|
export async function renderFile(inputPath, outputPath, threshold = 0.005) {
|
|
const resolvedInput = resolve(inputPath);
|
|
const resolvedOutput = resolve(outputPath);
|
|
const baseDir = dirname(resolvedInput);
|
|
|
|
console.log(`Reading : ${resolvedInput}`);
|
|
|
|
const { data, content: raw } = matter(readFileSync(resolvedInput, "utf8"));
|
|
const lines = raw.split("\n").map(parseLine);
|
|
|
|
threshold =
|
|
typeof data["dedup"] === "number" && data["dedup"] > 0
|
|
? data["dedup"]
|
|
: threshold;
|
|
console.log(`Threshold : ${threshold}`);
|
|
|
|
/** @type {Stats} */
|
|
const stats = { comparisons: 0, grouped: 0 };
|
|
const grouped = await groupImages(lines, baseDir, threshold, stats);
|
|
const output = render(grouped);
|
|
|
|
writeFileSync(resolvedOutput, matter.stringify(output, data), "utf8");
|
|
|
|
console.log(
|
|
`Comparisons: ${stats.comparisons} | grouped: ${stats.grouped}`,
|
|
);
|
|
console.log(`Wrote : ${resolvedOutput}`);
|
|
|
|
return stats;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// CLI entry-point (runs only when executed directly)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
const isMain =
|
|
process.argv[1] && resolve(process.argv[1]) === resolve(__filename);
|
|
|
|
if (isMain) {
|
|
const args = process.argv.slice(2);
|
|
const flagIdx = args.indexOf("--threshold");
|
|
const threshold = flagIdx !== -1 ? parseFloat(args[flagIdx + 1]) : 0.005;
|
|
const positional =
|
|
flagIdx === -1
|
|
? args
|
|
: args.filter(
|
|
(/** @type {string} */ _a, /** @type {number} */ i) =>
|
|
i !== flagIdx && i !== flagIdx + 1,
|
|
);
|
|
|
|
const inputPath = positional[0];
|
|
const outputPath = positional[1] || positional[0];
|
|
|
|
if (!inputPath) {
|
|
console.error(
|
|
"Usage: node group_similar.js <input.md> [output.md] [--threshold 0.005]",
|
|
);
|
|
process.exit(1);
|
|
}
|
|
|
|
await renderFile(inputPath, outputPath, threshold);
|
|
}
|