History

fix(core): parse favicon host via tldts

Use tldts domain parsing in favicon resolution and add its dependency.

Saatvik Arya committed 7 months ago

Commit 23e0a251 parent 62dd7413 files · +19 −58Browse files

Unified · Split

3 files
  1. bun.lock +12 −3
  2. packages/sources/core/package.json +2 −1
  3. packages/sources/core/src/favicon.ts +5 −54
bun.lock+12−3
@@ -19,7 +19,7 @@ · 18 unmodified lines
19 },19 },
20 "apps/executor": {20 "apps/executor": {
21 "name": "executor",21 "name": "executor",
22 "version": "1.3.0-beta.0",22 "version": "1.3.0-beta.3",
23 "bin": {23 "bin": {
24 "executor": "bin/executor",24 "executor": "bin/executor",
25 },25 },
@@ -424,6 +424,7 @@ · 398 unmodified lines
424 "@executor/ir": "workspace:*",424 "@executor/ir": "workspace:*",
425 "@modelcontextprotocol/sdk": "^1.26.0",425 "@modelcontextprotocol/sdk": "^1.26.0",
426 "effect": "catalog:",426 "effect": "catalog:",
427 "tldts": "^7.0.27",
427 },428 },
428 "devDependencies": {429 "devDependencies": {
429 "@types/node": "catalog:",430 "@types/node": "catalog:",
@@ -2556,9 +2557,9 @@ · 2126 unmodified lines
2556 2557
2557 "tinyspy": ["tinyspy@4.0.4", "", {}, "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q=="],2558 "tinyspy": ["tinyspy@4.0.4", "", {}, "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q=="],
2558 2559
2559 "tldts": ["tldts@7.0.25", "", { "dependencies": { "tldts-core": "^7.0.25" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-keinCnPbwXEUG3ilrWQZU+CqcTTzHq9m2HhoUP2l7Xmi8l1LuijAXLpAJ5zRW+ifKTNscs4NdCkfkDCBYm352w=="],2560 "tldts": ["tldts@7.0.27", "", { "dependencies": { "tldts-core": "^7.0.27" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-I4FZcVFcqCRuT0ph6dCDpPuO4Xgzvh+spkcTr1gK7peIvxWauoloVO0vuy1FQnijT63ss6AsHB6+OIM4aXHbPg=="],
2560 2561
2561 "tldts-core": ["tldts-core@7.0.25", "", {}, "sha512-ZjCZK0rppSBu7rjHYDYsEaMOIbbT+nWF57hKkv4IUmZWBNrBWBOjIElc0mKRgLM8bm7x/BBlof6t2gi/Oq/Asw=="],2562 "tldts-core": ["tldts-core@7.0.27", "", {}, "sha512-YQ7uPjgWUibIK6DW5lrKujGwUKhLevU4hcGbP5O6TcIUb+oTjJYJVWPS4nZsIHrEEEG6myk/oqAJUEQmpZrHsg=="],
2562 2563
2563 "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="],2564 "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="],
2564 2565
@@ -2728,6 +2729,8 @@ · 163 unmodified lines
2728 2729
2729 "@dotenvx/dotenvx/which": ["which@4.0.0", "", { "dependencies": { "isexe": "^3.1.1" }, "bin": { "node-which": "bin/which.js" } }, "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg=="],2730 "@dotenvx/dotenvx/which": ["which@4.0.0", "", { "dependencies": { "isexe": "^3.1.1" }, "bin": { "node-which": "bin/which.js" } }, "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg=="],
2730 2731
2732 "@executor/web/tldts": ["tldts@7.0.25", "", { "dependencies": { "tldts-core": "^7.0.25" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-keinCnPbwXEUG3ilrWQZU+CqcTTzHq9m2HhoUP2l7Xmi8l1LuijAXLpAJ5zRW+ifKTNscs4NdCkfkDCBYm352w=="],
2733
2731 "@manypkg/find-root/@types/node": ["@types/node@12.20.55", "", {}, "sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ=="],2734 "@manypkg/find-root/@types/node": ["@types/node@12.20.55", "", {}, "sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ=="],
2732 2735
2733 "@manypkg/find-root/fs-extra": ["fs-extra@8.1.0", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^4.0.0", "universalify": "^0.1.0" } }, "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g=="],2736 "@manypkg/find-root/fs-extra": ["fs-extra@8.1.0", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^4.0.0", "universalify": "^0.1.0" } }, "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g=="],
@@ -2818,10 +2821,14 @@ · 84 unmodified lines
2818 2821
2819 "@dotenvx/dotenvx/which/isexe": ["isexe@3.1.5", "", {}, "sha512-6B3tLtFqtQS4ekarvLVMZ+X+VlvQekbe4taUkf/rhVO3d/h0M2rfARm/pXLcPEsjjMsFgrFgSrhQIxcSVrBz8w=="],2822 "@dotenvx/dotenvx/which/isexe": ["isexe@3.1.5", "", {}, "sha512-6B3tLtFqtQS4ekarvLVMZ+X+VlvQekbe4taUkf/rhVO3d/h0M2rfARm/pXLcPEsjjMsFgrFgSrhQIxcSVrBz8w=="],
2820 2823
2824 "@executor/web/tldts/tldts-core": ["tldts-core@7.0.25", "", {}, "sha512-ZjCZK0rppSBu7rjHYDYsEaMOIbbT+nWF57hKkv4IUmZWBNrBWBOjIElc0mKRgLM8bm7x/BBlof6t2gi/Oq/Asw=="],
2825
2821 "@ts-morph/common/minimatch/brace-expansion": ["brace-expansion@5.0.5", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ=="],2826 "@ts-morph/common/minimatch/brace-expansion": ["brace-expansion@5.0.5", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ=="],
2822 2827
2823 "cliui/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],2828 "cliui/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
2824 2829
2830 "msw/tough-cookie/tldts": ["tldts@7.0.25", "", { "dependencies": { "tldts-core": "^7.0.25" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-keinCnPbwXEUG3ilrWQZU+CqcTTzHq9m2HhoUP2l7Xmi8l1LuijAXLpAJ5zRW+ifKTNscs4NdCkfkDCBYm352w=="],
2831
2825 "ora/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="],2832 "ora/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="],
2826 2833
2827 "read-yaml-file/js-yaml/argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="],2834 "read-yaml-file/js-yaml/argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="],
@@ -2839,5 +2846,7 @@ · 11 unmodified lines
2839 "yargs/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],2846 "yargs/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
2840 2847
2841 "@ts-morph/common/minimatch/brace-expansion/balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="],2848 "@ts-morph/common/minimatch/brace-expansion/balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="],
2849
2850 "msw/tough-cookie/tldts/tldts-core": ["tldts-core@7.0.25", "", {}, "sha512-ZjCZK0rppSBu7rjHYDYsEaMOIbbT+nWF57hKkv4IUmZWBNrBWBOjIElc0mKRgLM8bm7x/BBlof6t2gi/Oq/Asw=="],
2842 }2851 }
2843}2852}
packages/sources/core/package.json+2−1
@@ -15,7 +15,8 @@ · 14 unmodified lines
15 "@executor/codemode-core": "workspace:*",15 "@executor/codemode-core": "workspace:*",
16 "@executor/ir": "workspace:*",16 "@executor/ir": "workspace:*",
17 "@modelcontextprotocol/sdk": "^1.26.0",17 "@modelcontextprotocol/sdk": "^1.26.0",
18 "effect": "catalog:"18 "effect": "catalog:",
19 "tldts": "^7.0.27"
19 },20 },
20 "devDependencies": {21 "devDependencies": {
21 "@types/node": "catalog:",22 "@types/node": "catalog:",
packages/sources/core/src/favicon.ts+5−54
@@ -1,16 +1,4 @@
1const COMMON_COMPOUND_SUFFIXES = new Set([1import { parse as parseDomain } from "tldts";
2 "ac.uk",
3 "co.in",
4 "co.jp",
5 "co.nz",
6 "co.uk",
7 "com.au",
8 "com.br",
9 "com.mx",
10 "net.au",
11 "org.au",
12 "org.uk",
13]);
14 2
15const RAW_HOSTS = new Set([3const RAW_HOSTS = new Set([
16 "cdn.jsdelivr.net",4 "cdn.jsdelivr.net",
@@ -19,49 +7,13 @@ const RAW_HOSTS = new Set([ · 2 unmodified lines
19 "unpkg.com",7 "unpkg.com",
20]);8]);
21 9
22function trimOrNull(value: string | null | undefined): string | null {
23 if (value == null) {
24 return null;
25 }
26
27 const trimmed = value.trim();
28 return trimmed.length > 0 ? trimmed : null;
29}
30
31function isIpv4Address(value: string): boolean {
32 return /^\d{1,3}(?:\.\d{1,3}){3}$/.test(value);
33}
34
35export function getRegistrableDomain(hostname: string): string | null {
36 const normalized = hostname.trim().toLowerCase().replace(/^\.+|\.+$/g, "");
37 if (!normalized) {
38 return null;
39 }
40
41 if (normalized === "localhost" || isIpv4Address(normalized)) {
42 return normalized;
43 }
44
45 const parts = normalized.split(".").filter((part) => part.length > 0);
46 if (parts.length < 2) {
47 return null;
48 }
49
50 const suffix = parts.slice(-2).join(".");
51 if (parts.length >= 3 && COMMON_COMPOUND_SUFFIXES.has(suffix)) {
52 return parts.slice(-3).join(".");
53 }
54
55 return parts.slice(-2).join(".");
56}
57
58export function getFaviconUrlForRemoteUrl(10export function getFaviconUrlForRemoteUrl(
59 value: string | null | undefined,11 value: string | null | undefined,
60 options: {12 options: {
61 allowRawHosts?: boolean;13 allowRawHosts?: boolean;
62 } = {},14 } = {},
63): string | null {15): string | null {
64 const trimmed = trimOrNull(value);16 const trimmed = value?.trim();
65 if (!trimmed) {17 if (!trimmed) {
66 return null;18 return null;
67 }19 }
@@ -76,10 +28,9 @@ export function getFaviconUrlForRemoteUrl( · 8 unmodified lines
76 return null;28 return null;
77 }29 }
78 30
79 const domain = getRegistrableDomain(url.hostname);31 const parsed = parseDomain(url.hostname);
80 return domain32 const domain = parsed.domain ?? url.hostname;
81 ? `https://www.google.com/s2/favicons?domain=${encodeURIComponent(domain)}&sz=32`33 return `https://www.google.com/s2/favicons?domain=${encodeURIComponent(domain)}&sz=32`;
82 : null;
83 } catch {34 } catch {
84 return null;35 return null;
85 }36 }