Fix spot dedupe: merge same-named spots across geohash/city + name-suffix variants

The old identity dedupe keyed on `name|place` where place was a single field
(geohash OR location OR city, first non-empty). Two copies of the same spot —
one carrying a geohash, one carrying only a city — produced different keys and
both survived (the Yellow Park duplicate).

Now:
- Name is reduced to a signature that strips trailing generic words
  (skatepark/park/plaza/spot/diy/...) and leading articles, so "Yellow Park",
  "Yellow Park Skatepark" and "Yellow Park " collapse to "yellow".
- Places are compared on human-readable tokens (location/city/country) only;
  the opaque geohash is excluded. An empty place on either side is treated as a
  wildcard, so a geohash-only copy still merges with a "Bangkok" copy.
- Genuinely different spots (different name signatures, or non-empty places with
  no shared token) are still kept separate; single-word names are preserved so
  generic names don't over-collapse.

Co-authored-by: shakespeare.diy <assistant@shakespeare.diy>
This commit is contained in:
shakespeare.diy 2026-09-09 23:56:00 +07:00
parent 7e7cf0a8bf
commit ee23f3e063

View file

@ -224,27 +224,93 @@ function placeMatches(candidate: string, query: string): boolean {
.some((word) => word.length >= 3 && editDistance(word, q, 1) <= 1);
}
/**
* Reduce a spot name to a stable dedupe signature: lowercased, punctuation
* stripped, and common generic suffixes ("skatepark", "park", "plaza", "spot",
* "diy", etc.) removed — so "Yellow Park", "Yellow Park Skatepark" and
* "Yellow Park " all collapse to the same "yellow" core. Kept conservative:
* we only strip trailing generic words, never interior ones.
*/
const GENERIC_SPOT_WORDS = new Set([
'skatepark', 'skateplaza', 'skate', 'park', 'plaza', 'spot', 'diy',
'bowl', 'ramp', 'the', 'a',
]);
function spotNameSignature(name: string): string {
const words = normalizeForDedupe(name)
.replace(/[^a-z0-9 ]/gi, '')
.split(' ')
.filter(Boolean);
// Drop trailing generic words ("yellow park skatepark" → "yellow").
while (words.length > 1 && GENERIC_SPOT_WORDS.has(words[words.length - 1])) {
words.pop();
}
// Also drop leading articles ("the yellow" → "yellow").
while (words.length > 1 && GENERIC_SPOT_WORDS.has(words[0])) {
words.shift();
}
return words.join(' ');
}
/**
* Collapse spots that are the SAME real-world spot published as different
* addressable events — e.g. the same place re-posted under a new `d` identifier
* or by two different accounts. Coordinate dedupe (kind:pubkey:d) can't catch
* these because the coordinates differ, so we also dedupe on a
* name + location/geohash signature and keep the freshest event.
* these because the coordinates differ.
*
* Two spots merge when their NAME signatures match AND their places are
* compatible. "Compatible" is deliberately loose: a spot with a geohash and a
* spot with only a city name should still merge if the names match, because the
* same real-world spot is often posted once with precise coords and once with
* just "Bangkok". So we treat an EMPTY place on either side as a wildcard, and
* otherwise require the two place strings to share a token (e.g. both mention
* "bangkok"). This is what was letting Yellow Park duplicate: one copy carried a
* geohash, the other a city, so the old geohash-vs-city keys never matched.
*/
function deduplicateByIdentity<T extends { tags: string[][]; created_at: number }>(spots: T[]): T[] {
const seen = new Map<string, T>();
const kept: T[] = [];
// Human-readable place tokens only (city/location/country) — NOT the geohash,
// which is an opaque code that would never token-match a city name.
const placeTokens = (e: T): Set<string> => {
const raw = [getTag(e, 'location'), getTag(e, 'city'), getTag(e, 'country')]
.map((v) => normalizeForDedupe(v))
.filter(Boolean)
.join(' ');
return new Set(raw.split(/[^a-z0-9]+/i).filter((t) => t.length >= 3));
};
const placesCompatible = (a: Set<string>, b: Set<string>): boolean => {
// Empty human-place on either side → can't contradict (e.g. one copy only
// carried a geohash). Treat as compatible so a geohash-only copy still
// merges with a "Bangkok" copy of the same-named spot.
if (a.size === 0 || b.size === 0) return true;
for (const t of a) if (b.has(t)) return true;
return false;
};
for (const e of spots) {
const name = normalizeForDedupe(getTag(e, 'title') ?? getTag(e, 'name'));
// Prefer geohash (precise) then location text as the place signature.
const place = normalizeForDedupe(getTag(e, 'g') ?? getTag(e, 'location') ?? getTag(e, 'city'));
// No name → can't confidently match; keep it (unique key on coordinate).
const key = name ? `${name}|${place}` : `__unique__${buildNaddr(e as unknown as { kind: number; pubkey: string; tags: string[][] })}`;
const existing = seen.get(key);
if (!existing || e.created_at > existing.created_at) {
seen.set(key, e);
const sig = spotNameSignature(getTag(e, 'title') ?? getTag(e, 'name') ?? '');
// No usable name → can't confidently merge; always keep it.
if (!sig) {
kept.push(e);
continue;
}
const tokens = placeTokens(e);
const dupIdx = kept.findIndex(
(k) =>
spotNameSignature(getTag(k, 'title') ?? getTag(k, 'name') ?? '') === sig &&
placesCompatible(placeTokens(k), tokens)
);
if (dupIdx === -1) {
kept.push(e);
} else if (e.created_at > kept[dupIdx].created_at) {
// Same spot, newer copy → replace, preferring the richer place info.
kept[dupIdx] = e;
}
}
return Array.from(seen.values());
return kept;
}
function buildNaddr(event: { kind: number; pubkey: string; tags: string[][] }): string {