add video; update dedup script

This commit is contained in:
polv committed 2026-04-16 15:40:43 +07:00
1 parent b85cf6053b
commit 6cff0efff2
3 files changed
+1895 -3

No files matched your search

+1
View File
@@ -9,6 +9,7 @@ hero:
<Downloader />
- [东非最发达的城市现在怎么样了?揭秘肯尼亚不为人知的一面](./东非最发达的城市现在怎么样了?揭秘肯尼亚不为人知的一面.md)
- [一次性搞清楚北方四岛 | 毛泽东日本代表团谈话 | 中苏交恶](./一次性搞清楚北方四岛_|_毛泽东日本代表团谈话_|_中苏交恶.md)
- [对非零关税:补齐中国资源短板,带动中非共同发展](./对非零关税:补齐中国资源短板,带动中非共同发展.md)
- [馬來西亞為何力挺伊朗?當宗教之外的現實利益,讓遜尼派國家站在什葉派這邊](./馬來西亞為何力挺伊朗?當宗教之外的現實利益,讓遜尼派國家站在什葉派這邊.md)
+9 -3
View File
@@ -25,6 +25,7 @@
import { readFileSync, writeFileSync } from "node:fs";
import { resolve, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import matter from "gray-matter";
// @ts-ignore - sharp ships no bundled .d.ts; types are not needed at runtime
import sharp from "sharp";
@@ -267,17 +268,22 @@ export async function renderFile(inputPath, outputPath, threshold = 0.005) {
const baseDir = dirname(resolvedInput);
console.log(`Reading : ${resolvedInput}`);
console.log(`Threshold : ${threshold}`);
const raw = readFileSync(resolvedInput, "utf8");
const { data, content: raw } = matter(readFileSync(resolvedInput, "utf8"));
const lines = raw.split("\n").map(parseLine);
threshold =
typeof data["dedup"] === "number" && data["dedup"] > 0
? data["dedup"]
: threshold;
console.log(`Threshold : ${threshold}`);
/** @type {Stats} */
const stats = { comparisons: 0, grouped: 0 };
const grouped = await groupImages(lines, baseDir, threshold, stats);
const output = render(grouped);
writeFileSync(resolvedOutput, output, "utf8");
writeFileSync(resolvedOutput, matter.stringify(output, data), "utf8");
console.log(
`Comparisons: ${stats.comparisons} | grouped: ${stats.grouped}`,