From ee23f3e0637ff70bd45d60090621f5d2f1acd492 Mon Sep 17 00:00:00 2001 From: "shakespeare.diy" Date: Wed, 9 Sep 2026 23:56:00 +0700 Subject: [PATCH] Fix spot dedupe: merge same-named spots across geohash/city + name-suffix variants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old identity dedupe keyed on `name|place` where place was a single field (geohash OR location OR city, first non-empty). Two copies of the same spot — one carrying a geohash, one carrying only a city — produced different keys and both survived (the Yellow Park duplicate). Now: - Name is reduced to a signature that strips trailing generic words (skatepark/park/plaza/spot/diy/...) and leading articles, so "Yellow Park", "Yellow Park Skatepark" and "Yellow Park " collapse to "yellow". - Places are compared on human-readable tokens (location/city/country) only; the opaque geohash is excluded. An empty place on either side is treated as a wildcard, so a geohash-only copy still merges with a "Bangkok" copy. - Genuinely different spots (different name signatures, or non-empty places with no shared token) are still kept separate; single-word names are preserved so generic names don't over-collapse. Co-authored-by: shakespeare.diy --- src/hooks/useCVMClient.ts | 90 +++++++++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 12 deletions(-) diff --git a/src/hooks/useCVMClient.ts b/src/hooks/useCVMClient.ts index 5025d40..c5af238 100644 --- a/src/hooks/useCVMClient.ts +++ b/src/hooks/useCVMClient.ts @@ -224,27 +224,93 @@ function placeMatches(candidate: string, query: string): boolean { .some((word) => word.length >= 3 && editDistance(word, q, 1) <= 1); } +/** + * Reduce a spot name to a stable dedupe signature: lowercased, punctuation + * stripped, and common generic suffixes ("skatepark", "park", "plaza", "spot", + * "diy", etc.) removed — so "Yellow Park", "Yellow Park Skatepark" and + * "Yellow Park " all collapse to the same "yellow" core. Kept conservative: + * we only strip trailing generic words, never interior ones. + */ +const GENERIC_SPOT_WORDS = new Set([ + 'skatepark', 'skateplaza', 'skate', 'park', 'plaza', 'spot', 'diy', + 'bowl', 'ramp', 'the', 'a', +]); + +function spotNameSignature(name: string): string { + const words = normalizeForDedupe(name) + .replace(/[^a-z0-9 ]/gi, '') + .split(' ') + .filter(Boolean); + // Drop trailing generic words ("yellow park skatepark" → "yellow"). + while (words.length > 1 && GENERIC_SPOT_WORDS.has(words[words.length - 1])) { + words.pop(); + } + // Also drop leading articles ("the yellow" → "yellow"). + while (words.length > 1 && GENERIC_SPOT_WORDS.has(words[0])) { + words.shift(); + } + return words.join(' '); +} + /** * Collapse spots that are the SAME real-world spot published as different * addressable events — e.g. the same place re-posted under a new `d` identifier * or by two different accounts. Coordinate dedupe (kind:pubkey:d) can't catch - * these because the coordinates differ, so we also dedupe on a - * name + location/geohash signature and keep the freshest event. + * these because the coordinates differ. + * + * Two spots merge when their NAME signatures match AND their places are + * compatible. "Compatible" is deliberately loose: a spot with a geohash and a + * spot with only a city name should still merge if the names match, because the + * same real-world spot is often posted once with precise coords and once with + * just "Bangkok". So we treat an EMPTY place on either side as a wildcard, and + * otherwise require the two place strings to share a token (e.g. both mention + * "bangkok"). This is what was letting Yellow Park duplicate: one copy carried a + * geohash, the other a city, so the old geohash-vs-city keys never matched. */ function deduplicateByIdentity(spots: T[]): T[] { - const seen = new Map(); + const kept: T[] = []; + + // Human-readable place tokens only (city/location/country) — NOT the geohash, + // which is an opaque code that would never token-match a city name. + const placeTokens = (e: T): Set => { + const raw = [getTag(e, 'location'), getTag(e, 'city'), getTag(e, 'country')] + .map((v) => normalizeForDedupe(v)) + .filter(Boolean) + .join(' '); + return new Set(raw.split(/[^a-z0-9]+/i).filter((t) => t.length >= 3)); + }; + + const placesCompatible = (a: Set, b: Set): boolean => { + // Empty human-place on either side → can't contradict (e.g. one copy only + // carried a geohash). Treat as compatible so a geohash-only copy still + // merges with a "Bangkok" copy of the same-named spot. + if (a.size === 0 || b.size === 0) return true; + for (const t of a) if (b.has(t)) return true; + return false; + }; + for (const e of spots) { - const name = normalizeForDedupe(getTag(e, 'title') ?? getTag(e, 'name')); - // Prefer geohash (precise) then location text as the place signature. - const place = normalizeForDedupe(getTag(e, 'g') ?? getTag(e, 'location') ?? getTag(e, 'city')); - // No name → can't confidently match; keep it (unique key on coordinate). - const key = name ? `${name}|${place}` : `__unique__${buildNaddr(e as unknown as { kind: number; pubkey: string; tags: string[][] })}`; - const existing = seen.get(key); - if (!existing || e.created_at > existing.created_at) { - seen.set(key, e); + const sig = spotNameSignature(getTag(e, 'title') ?? getTag(e, 'name') ?? ''); + // No usable name → can't confidently merge; always keep it. + if (!sig) { + kept.push(e); + continue; + } + const tokens = placeTokens(e); + const dupIdx = kept.findIndex( + (k) => + spotNameSignature(getTag(k, 'title') ?? getTag(k, 'name') ?? '') === sig && + placesCompatible(placeTokens(k), tokens) + ); + if (dupIdx === -1) { + kept.push(e); + } else if (e.created_at > kept[dupIdx].created_at) { + // Same spot, newer copy → replace, preferring the richer place info. + kept[dupIdx] = e; } } - return Array.from(seen.values()); + + return kept; } function buildNaddr(event: { kind: number; pubkey: string; tags: string[][] }): string {