History

fix(core): parse favicon host via tldts

Use tldts domain parsing in favicon resolution and add its dependency.

Saatvik Arya committed 7 months ago

Commit 23e0a251 parent 62dd7413 files · +19 −58Browse files

Unified · Split

3 files
  1. bun.lock +12 −3
  2. packages/sources/core/package.json +2 −1
  3. packages/sources/core/src/favicon.ts +5 −54
bun.lock+12−3
@@ -19,7 +19,7 @@ · 18 unmodified lines
1919 },
2020 "apps/executor": {
2121 "name": "executor",
22− "version": "1.3.0-beta.0",
22+ "version": "1.3.0-beta.3",
2323 "bin": {
2424 "executor": "bin/executor",
2525 },
@@ -424,6 +424,7 @@ · 398 unmodified lines
424424 "@executor/ir": "workspace:*",
425425 "@modelcontextprotocol/sdk": "^1.26.0",
426426 "effect": "catalog:",
427+ "tldts": "^7.0.27",
427428 },
428429 "devDependencies": {
429430 "@types/node": "catalog:",
@@ -2556,9 +2557,9 @@ · 2126 unmodified lines
25562557
25572558 "tinyspy": ["tinyspy@4.0.4", "", {}, "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q=="],
25582559
2559− "tldts": ["tldts@7.0.25", "", { "dependencies": { "tldts-core": "^7.0.25" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-keinCnPbwXEUG3ilrWQZU+CqcTTzHq9m2HhoUP2l7Xmi8l1LuijAXLpAJ5zRW+ifKTNscs4NdCkfkDCBYm352w=="],
2560+ "tldts": ["tldts@7.0.27", "", { "dependencies": { "tldts-core": "^7.0.27" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-I4FZcVFcqCRuT0ph6dCDpPuO4Xgzvh+spkcTr1gK7peIvxWauoloVO0vuy1FQnijT63ss6AsHB6+OIM4aXHbPg=="],
25602561
2561− "tldts-core": ["tldts-core@7.0.25", "", {}, "sha512-ZjCZK0rppSBu7rjHYDYsEaMOIbbT+nWF57hKkv4IUmZWBNrBWBOjIElc0mKRgLM8bm7x/BBlof6t2gi/Oq/Asw=="],
2562+ "tldts-core": ["tldts-core@7.0.27", "", {}, "sha512-YQ7uPjgWUibIK6DW5lrKujGwUKhLevU4hcGbP5O6TcIUb+oTjJYJVWPS4nZsIHrEEEG6myk/oqAJUEQmpZrHsg=="],
25622563
25632564 "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="],
25642565
@@ -2728,6 +2729,8 @@ · 163 unmodified lines
27282729
27292730 "@dotenvx/dotenvx/which": ["which@4.0.0", "", { "dependencies": { "isexe": "^3.1.1" }, "bin": { "node-which": "bin/which.js" } }, "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg=="],
27302731
2732+ "@executor/web/tldts": ["tldts@7.0.25", "", { "dependencies": { "tldts-core": "^7.0.25" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-keinCnPbwXEUG3ilrWQZU+CqcTTzHq9m2HhoUP2l7Xmi8l1LuijAXLpAJ5zRW+ifKTNscs4NdCkfkDCBYm352w=="],
2733+
27312734 "@manypkg/find-root/@types/node": ["@types/node@12.20.55", "", {}, "sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ=="],
27322735
27332736 "@manypkg/find-root/fs-extra": ["fs-extra@8.1.0", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^4.0.0", "universalify": "^0.1.0" } }, "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g=="],
@@ -2818,10 +2821,14 @@ · 84 unmodified lines
28182821
28192822 "@dotenvx/dotenvx/which/isexe": ["isexe@3.1.5", "", {}, "sha512-6B3tLtFqtQS4ekarvLVMZ+X+VlvQekbe4taUkf/rhVO3d/h0M2rfARm/pXLcPEsjjMsFgrFgSrhQIxcSVrBz8w=="],
28202823
2824+ "@executor/web/tldts/tldts-core": ["tldts-core@7.0.25", "", {}, "sha512-ZjCZK0rppSBu7rjHYDYsEaMOIbbT+nWF57hKkv4IUmZWBNrBWBOjIElc0mKRgLM8bm7x/BBlof6t2gi/Oq/Asw=="],
2825+
28212826 "@ts-morph/common/minimatch/brace-expansion": ["brace-expansion@5.0.5", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ=="],
28222827
28232828 "cliui/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
28242829
2830+ "msw/tough-cookie/tldts": ["tldts@7.0.25", "", { "dependencies": { "tldts-core": "^7.0.25" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-keinCnPbwXEUG3ilrWQZU+CqcTTzHq9m2HhoUP2l7Xmi8l1LuijAXLpAJ5zRW+ifKTNscs4NdCkfkDCBYm352w=="],
2831+
28252832 "ora/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="],
28262833
28272834 "read-yaml-file/js-yaml/argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="],
@@ -2839,5 +2846,7 @@ · 11 unmodified lines
28392846 "yargs/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
28402847
28412848 "@ts-morph/common/minimatch/brace-expansion/balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="],
2849+
2850+ "msw/tough-cookie/tldts/tldts-core": ["tldts-core@7.0.25", "", {}, "sha512-ZjCZK0rppSBu7rjHYDYsEaMOIbbT+nWF57hKkv4IUmZWBNrBWBOjIElc0mKRgLM8bm7x/BBlof6t2gi/Oq/Asw=="],
28422851 }
28432852 }
packages/sources/core/package.json+2−1
@@ -15,7 +15,8 @@ · 14 unmodified lines
1515 "@executor/codemode-core": "workspace:*",
1616 "@executor/ir": "workspace:*",
1717 "@modelcontextprotocol/sdk": "^1.26.0",
18− "effect": "catalog:"
18+ "effect": "catalog:",
19+ "tldts": "^7.0.27"
1920 },
2021 "devDependencies": {
2122 "@types/node": "catalog:",
packages/sources/core/src/favicon.ts+5−54
@@ -1,16 +1,4 @@
1−const COMMON_COMPOUND_SUFFIXES = new Set([
2− "ac.uk",
3− "co.in",
4− "co.jp",
5− "co.nz",
6− "co.uk",
7− "com.au",
8− "com.br",
9− "com.mx",
10− "net.au",
11− "org.au",
12− "org.uk",
13−]);
1+import { parse as parseDomain } from "tldts";
142
153 const RAW_HOSTS = new Set([
164 "cdn.jsdelivr.net",
@@ -19,49 +7,13 @@ const RAW_HOSTS = new Set([ · 2 unmodified lines
197 "unpkg.com",
208 ]);
219
22−function trimOrNull(value: string | null | undefined): string | null {
23− if (value == null) {
24− return null;
25− }
26−
27− const trimmed = value.trim();
28− return trimmed.length > 0 ? trimmed : null;
29−}
30−
31−function isIpv4Address(value: string): boolean {
32− return /^\d{1,3}(?:\.\d{1,3}){3}$/.test(value);
33−}
34−
35−export function getRegistrableDomain(hostname: string): string | null {
36− const normalized = hostname.trim().toLowerCase().replace(/^\.+|\.+$/g, "");
37− if (!normalized) {
38− return null;
39− }
40−
41− if (normalized === "localhost" || isIpv4Address(normalized)) {
42− return normalized;
43− }
44−
45− const parts = normalized.split(".").filter((part) => part.length > 0);
46− if (parts.length < 2) {
47− return null;
48− }
49−
50− const suffix = parts.slice(-2).join(".");
51− if (parts.length >= 3 && COMMON_COMPOUND_SUFFIXES.has(suffix)) {
52− return parts.slice(-3).join(".");
53− }
54−
55− return parts.slice(-2).join(".");
56−}
57−
5810 export function getFaviconUrlForRemoteUrl(
5911 value: string | null | undefined,
6012 options: {
6113 allowRawHosts?: boolean;
6214 } = {},
6315 ): string | null {
64− const trimmed = trimOrNull(value);
16+ const trimmed = value?.trim();
6517 if (!trimmed) {
6618 return null;
6719 }
@@ -76,10 +28,9 @@ export function getFaviconUrlForRemoteUrl( · 8 unmodified lines
7628 return null;
7729 }
7830
79− const domain = getRegistrableDomain(url.hostname);
80− return domain
81− ? `https://www.google.com/s2/favicons?domain=${encodeURIComponent(domain)}&sz=32`
82− : null;
31+ const parsed = parseDomain(url.hostname);
32+ const domain = parsed.domain ?? url.hostname;
33+ return `https://www.google.com/s2/favicons?domain=${encodeURIComponent(domain)}&sz=32`;
8334 } catch {
8435 return null;
8536 }