Fix spot dedupe: merge same-named spots across geohash/city + name-suffix variants
The old identity dedupe keyed on `name|place` where place was a single field (geohash OR location OR city, first non-empty). Two copies of the same spot — one carrying a geohash, one carrying only a city — produced different keys and both survived (the Yellow Park duplicate). Now: - Name is reduced to a signature that strips trailing generic words (skatepark/park/plaza/spot/diy/...) and leading articles, so "Yellow Park", "Yellow Park Skatepark" and "Yellow Park " collapse to "yellow". - Places are compared on human-readable tokens (location/city/country) only; the opaque geohash is excluded. An empty place on either side is treated as a wildcard, so a geohash-only copy still merges with a "Bangkok" copy. - Genuinely different spots (different name signatures, or non-empty places with no shared token) are still kept separate; single-word names are preserved so generic names don't over-collapse. Co-authored-by: shakespeare.diy <assistant@shakespeare.diy>
This commit is contained in:
parent
7e7cf0a8bf
commit
ee23f3e063
1 changed files with 78 additions and 12 deletions
|
|
@ -224,27 +224,93 @@ function placeMatches(candidate: string, query: string): boolean {
|
||||||
.some((word) => word.length >= 3 && editDistance(word, q, 1) <= 1);
|
.some((word) => word.length >= 3 && editDistance(word, q, 1) <= 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reduce a spot name to a stable dedupe signature: lowercased, punctuation
|
||||||
|
* stripped, and common generic suffixes ("skatepark", "park", "plaza", "spot",
|
||||||
|
* "diy", etc.) removed — so "Yellow Park", "Yellow Park Skatepark" and
|
||||||
|
* "Yellow Park " all collapse to the same "yellow" core. Kept conservative:
|
||||||
|
* we only strip trailing generic words, never interior ones.
|
||||||
|
*/
|
||||||
|
const GENERIC_SPOT_WORDS = new Set([
|
||||||
|
'skatepark', 'skateplaza', 'skate', 'park', 'plaza', 'spot', 'diy',
|
||||||
|
'bowl', 'ramp', 'the', 'a',
|
||||||
|
]);
|
||||||
|
|
||||||
|
function spotNameSignature(name: string): string {
|
||||||
|
const words = normalizeForDedupe(name)
|
||||||
|
.replace(/[^a-z0-9 ]/gi, '')
|
||||||
|
.split(' ')
|
||||||
|
.filter(Boolean);
|
||||||
|
// Drop trailing generic words ("yellow park skatepark" → "yellow").
|
||||||
|
while (words.length > 1 && GENERIC_SPOT_WORDS.has(words[words.length - 1])) {
|
||||||
|
words.pop();
|
||||||
|
}
|
||||||
|
// Also drop leading articles ("the yellow" → "yellow").
|
||||||
|
while (words.length > 1 && GENERIC_SPOT_WORDS.has(words[0])) {
|
||||||
|
words.shift();
|
||||||
|
}
|
||||||
|
return words.join(' ');
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Collapse spots that are the SAME real-world spot published as different
|
* Collapse spots that are the SAME real-world spot published as different
|
||||||
* addressable events — e.g. the same place re-posted under a new `d` identifier
|
* addressable events — e.g. the same place re-posted under a new `d` identifier
|
||||||
* or by two different accounts. Coordinate dedupe (kind:pubkey:d) can't catch
|
* or by two different accounts. Coordinate dedupe (kind:pubkey:d) can't catch
|
||||||
* these because the coordinates differ, so we also dedupe on a
|
* these because the coordinates differ.
|
||||||
* name + location/geohash signature and keep the freshest event.
|
*
|
||||||
|
* Two spots merge when their NAME signatures match AND their places are
|
||||||
|
* compatible. "Compatible" is deliberately loose: a spot with a geohash and a
|
||||||
|
* spot with only a city name should still merge if the names match, because the
|
||||||
|
* same real-world spot is often posted once with precise coords and once with
|
||||||
|
* just "Bangkok". So we treat an EMPTY place on either side as a wildcard, and
|
||||||
|
* otherwise require the two place strings to share a token (e.g. both mention
|
||||||
|
* "bangkok"). This is what was letting Yellow Park duplicate: one copy carried a
|
||||||
|
* geohash, the other a city, so the old geohash-vs-city keys never matched.
|
||||||
*/
|
*/
|
||||||
function deduplicateByIdentity<T extends { tags: string[][]; created_at: number }>(spots: T[]): T[] {
|
function deduplicateByIdentity<T extends { tags: string[][]; created_at: number }>(spots: T[]): T[] {
|
||||||
const seen = new Map<string, T>();
|
const kept: T[] = [];
|
||||||
|
|
||||||
|
// Human-readable place tokens only (city/location/country) — NOT the geohash,
|
||||||
|
// which is an opaque code that would never token-match a city name.
|
||||||
|
const placeTokens = (e: T): Set<string> => {
|
||||||
|
const raw = [getTag(e, 'location'), getTag(e, 'city'), getTag(e, 'country')]
|
||||||
|
.map((v) => normalizeForDedupe(v))
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(' ');
|
||||||
|
return new Set(raw.split(/[^a-z0-9]+/i).filter((t) => t.length >= 3));
|
||||||
|
};
|
||||||
|
|
||||||
|
const placesCompatible = (a: Set<string>, b: Set<string>): boolean => {
|
||||||
|
// Empty human-place on either side → can't contradict (e.g. one copy only
|
||||||
|
// carried a geohash). Treat as compatible so a geohash-only copy still
|
||||||
|
// merges with a "Bangkok" copy of the same-named spot.
|
||||||
|
if (a.size === 0 || b.size === 0) return true;
|
||||||
|
for (const t of a) if (b.has(t)) return true;
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
for (const e of spots) {
|
for (const e of spots) {
|
||||||
const name = normalizeForDedupe(getTag(e, 'title') ?? getTag(e, 'name'));
|
const sig = spotNameSignature(getTag(e, 'title') ?? getTag(e, 'name') ?? '');
|
||||||
// Prefer geohash (precise) then location text as the place signature.
|
// No usable name → can't confidently merge; always keep it.
|
||||||
const place = normalizeForDedupe(getTag(e, 'g') ?? getTag(e, 'location') ?? getTag(e, 'city'));
|
if (!sig) {
|
||||||
// No name → can't confidently match; keep it (unique key on coordinate).
|
kept.push(e);
|
||||||
const key = name ? `${name}|${place}` : `__unique__${buildNaddr(e as unknown as { kind: number; pubkey: string; tags: string[][] })}`;
|
continue;
|
||||||
const existing = seen.get(key);
|
}
|
||||||
if (!existing || e.created_at > existing.created_at) {
|
const tokens = placeTokens(e);
|
||||||
seen.set(key, e);
|
const dupIdx = kept.findIndex(
|
||||||
|
(k) =>
|
||||||
|
spotNameSignature(getTag(k, 'title') ?? getTag(k, 'name') ?? '') === sig &&
|
||||||
|
placesCompatible(placeTokens(k), tokens)
|
||||||
|
);
|
||||||
|
if (dupIdx === -1) {
|
||||||
|
kept.push(e);
|
||||||
|
} else if (e.created_at > kept[dupIdx].created_at) {
|
||||||
|
// Same spot, newer copy → replace, preferring the richer place info.
|
||||||
|
kept[dupIdx] = e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return Array.from(seen.values());
|
|
||||||
|
return kept;
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildNaddr(event: { kind: number; pubkey: string; tags: string[][] }): string {
|
function buildNaddr(event: { kind: number; pubkey: string; tags: string[][] }): string {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue