This commit is contained in:
Andres Resiri
@@ -0,0 +1,15 @@
|
||||
-- ============================================================
|
||||
-- MISSING INDEXES — Run in Neon SQL Editor
|
||||
-- ============================================================
|
||||
-- These indexes exist in prisma/schema.prisma but are NOT on the
|
||||
-- production database (lost during schema conflict resolution).
|
||||
-- CONCURRENTLY ensures no table locks during creation.
|
||||
-- ============================================================
|
||||
|
||||
-- SiteEvent: admin Signal Overview cross-brand queries
|
||||
-- Without this, groupBy on 30M rows scans the entire table
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "SiteEvent_eventType_timestamp_idx"
|
||||
ON "SiteEvent" ("eventType", "timestamp");
|
||||
|
||||
-- Verify:
|
||||
-- SELECT indexname FROM pg_indexes WHERE tablename = 'SiteEvent';
|
||||
@@ -0,0 +1,349 @@
|
||||
/**
|
||||
* Phase 1 — Metric Ground-Truth Audit
|
||||
* ───────────────────────────────────────
|
||||
*
|
||||
* Run against a live DB to capture the *actual* counts per metric
|
||||
* table for a single brand. Output is written to stdout AND to
|
||||
* scripts/audit-results.md so the ground truth lives alongside the
|
||||
* code-side audit.
|
||||
*
|
||||
* Usage:
|
||||
* DATABASE_URL=postgres://... npx tsx scripts/audit-phase1.ts
|
||||
* # or pick a different brand
|
||||
* BRAND_NAME="Modern Heart" npx tsx scripts/audit-phase1.ts
|
||||
*
|
||||
* This script is READ-ONLY — it only runs aggregate() / groupBy()
|
||||
* / count() queries. Safe to run in production.
|
||||
*
|
||||
* Scoping note: some metric tables are keyed by brandId directly
|
||||
* (SiteConversion, SiteEvent, RealUserMetric, AiInteraction,
|
||||
* AiUsageLog, PlatformEvent, MetricSnapshot, GscPage, TechnicalAudit)
|
||||
* while TrackedEvent and TrackedSession are scoped via TrackedSite.
|
||||
* The script resolves the brand's TrackedSite once and uses
|
||||
* trackedSiteId for those two tables.
|
||||
*/
|
||||
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
import { writeFileSync, mkdirSync } from "fs";
|
||||
import { dirname } from "path";
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
|
||||
// Accumulator for the markdown report. Every log() call appends to
|
||||
// both stdout and this buffer; the buffer is flushed to
|
||||
// scripts/audit-results.md at the end of the run.
|
||||
const report: string[] = [];
|
||||
function log(line: string) {
|
||||
log(line);
|
||||
report.push(line);
|
||||
}
|
||||
|
||||
// JSON.stringify replacer that handles BigInt + Date cleanly.
|
||||
function replacer(_key: string, value: unknown): unknown {
|
||||
if (typeof value === "bigint") return value.toString();
|
||||
if (value instanceof Date) return value.toISOString();
|
||||
return value;
|
||||
}
|
||||
|
||||
function j(v: unknown): string {
|
||||
return JSON.stringify(v, replacer);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const brandNeedle = process.env.BRAND_NAME ?? "Modern Heart";
|
||||
|
||||
const brand = await prisma.brand.findFirst({
|
||||
where: { name: { contains: brandNeedle, mode: "insensitive" } },
|
||||
select: { id: true, name: true, domain: true, createdAt: true },
|
||||
});
|
||||
if (!brand) {
|
||||
console.error(`No brand matching "${brandNeedle}" found.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const site = await prisma.trackedSite.findUnique({
|
||||
where: { brandId: brand.id },
|
||||
select: { id: true, status: true, firstEventAt: true, lastEventAt: true, createdAt: true },
|
||||
});
|
||||
|
||||
log("=== Brand ===");
|
||||
log(j({ brand }));
|
||||
log("=== TrackedSite ===");
|
||||
log(j({ site }));
|
||||
|
||||
// ─── Direct-brandId tables ──────────────────────────────────
|
||||
//
|
||||
// Each aggregate runs independently so a single failing query
|
||||
// doesn't blackhole the rest of the audit. Empty / errored rows
|
||||
// render as ERROR in the output.
|
||||
const brandTables: Array<{ name: string; run: () => Promise<unknown> }> = [
|
||||
{
|
||||
name: "SiteConversion",
|
||||
run: () => prisma.siteConversion.aggregate({
|
||||
where: { brandId: brand.id },
|
||||
_count: true,
|
||||
_min: { timestamp: true },
|
||||
_max: { timestamp: true },
|
||||
}),
|
||||
},
|
||||
{
|
||||
name: "SiteEvent",
|
||||
run: () => prisma.siteEvent.aggregate({
|
||||
where: { brandId: brand.id },
|
||||
_count: true,
|
||||
_min: { timestamp: true },
|
||||
_max: { timestamp: true },
|
||||
}),
|
||||
},
|
||||
{
|
||||
name: "RealUserMetric",
|
||||
run: () => prisma.realUserMetric.aggregate({
|
||||
where: { brandId: brand.id },
|
||||
_count: true,
|
||||
_min: { timestamp: true },
|
||||
_max: { timestamp: true },
|
||||
}),
|
||||
},
|
||||
{
|
||||
name: "AiInteraction",
|
||||
run: () => prisma.aiInteraction.aggregate({
|
||||
where: { brandId: brand.id },
|
||||
_count: true,
|
||||
_min: { timestamp: true },
|
||||
_max: { timestamp: true },
|
||||
}),
|
||||
},
|
||||
{
|
||||
name: "AiUsageLog",
|
||||
run: () => prisma.aiUsageLog.aggregate({
|
||||
where: { brandId: brand.id },
|
||||
_count: true,
|
||||
_min: { timestamp: true },
|
||||
_max: { timestamp: true },
|
||||
}),
|
||||
},
|
||||
{
|
||||
name: "PlatformEvent",
|
||||
run: () => prisma.platformEvent.aggregate({
|
||||
where: { brandId: brand.id },
|
||||
_count: true,
|
||||
_min: { timestamp: true },
|
||||
_max: { timestamp: true },
|
||||
}),
|
||||
},
|
||||
{
|
||||
name: "MetricSnapshot",
|
||||
run: () => prisma.metricSnapshot.aggregate({
|
||||
where: { brandId: brand.id },
|
||||
_count: true,
|
||||
_min: { date: true },
|
||||
_max: { date: true },
|
||||
}),
|
||||
},
|
||||
{
|
||||
name: "GscPage",
|
||||
run: () => prisma.gscPage.aggregate({
|
||||
where: { brandId: brand.id },
|
||||
_count: true,
|
||||
_min: { date: true },
|
||||
_max: { date: true },
|
||||
}),
|
||||
},
|
||||
{
|
||||
name: "TechnicalAudit",
|
||||
run: () => prisma.technicalAudit.aggregate({
|
||||
where: { brandId: brand.id },
|
||||
_count: true,
|
||||
_min: { createdAt: true },
|
||||
_max: { createdAt: true },
|
||||
}),
|
||||
},
|
||||
];
|
||||
|
||||
log("\n=== Brand-scoped tables ===");
|
||||
for (const t of brandTables) {
|
||||
try {
|
||||
const result = await t.run();
|
||||
log(`${t.name}: ${j(result)}`);
|
||||
} catch (err) {
|
||||
log(`${t.name}: ERROR ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── TrackedSite-scoped tables ──────────────────────────────
|
||||
log("\n=== TrackedSite-scoped tables ===");
|
||||
if (!site) {
|
||||
log("TrackedEvent / TrackedSession: SKIPPED (no TrackedSite for brand)");
|
||||
} else {
|
||||
try {
|
||||
const te = await prisma.trackedEvent.aggregate({
|
||||
where: { trackedSiteId: site.id },
|
||||
_count: true,
|
||||
_min: { timestamp: true },
|
||||
_max: { timestamp: true },
|
||||
});
|
||||
log(`TrackedEvent: ${j(te)}`);
|
||||
} catch (err) {
|
||||
log(`TrackedEvent: ERROR ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
try {
|
||||
const ts = await prisma.trackedSession.aggregate({
|
||||
where: { trackedSiteId: site.id },
|
||||
_count: true,
|
||||
_min: { startedAt: true },
|
||||
_max: { startedAt: true },
|
||||
});
|
||||
log(`TrackedSession: ${j(ts)}`);
|
||||
} catch (err) {
|
||||
log(`TrackedSession: ERROR ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Type breakdowns (the numbers the UI actually renders) ──
|
||||
log("\n=== Distinct event-type counts ===");
|
||||
|
||||
if (site) {
|
||||
try {
|
||||
const trackedTypes = await prisma.trackedEvent.groupBy({
|
||||
by: ["eventType"],
|
||||
where: { trackedSiteId: site.id },
|
||||
_count: true,
|
||||
});
|
||||
trackedTypes.sort((a, b) => (b._count as unknown as number) - (a._count as unknown as number));
|
||||
// Top 30 only — longer lists hide the signal.
|
||||
log(`TrackedEvent types (top 30): ${j(trackedTypes.slice(0, 30))}`);
|
||||
log(`TrackedEvent distinct type count: ${trackedTypes.length}`);
|
||||
} catch (err) {
|
||||
log(`TrackedEvent types: ERROR ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const siteConvTypes = await prisma.siteConversion.groupBy({
|
||||
by: ["conversionType"],
|
||||
where: { brandId: brand.id },
|
||||
_count: true,
|
||||
});
|
||||
siteConvTypes.sort((a, b) => (b._count as unknown as number) - (a._count as unknown as number));
|
||||
// All SiteConversion types — there are rarely more than ~20
|
||||
// distinct values but capped at 30 for safety.
|
||||
log(`SiteConversion types (top 30): ${j(siteConvTypes.slice(0, 30))}`);
|
||||
log(`SiteConversion distinct type count: ${siteConvTypes.length}`);
|
||||
} catch (err) {
|
||||
log(`SiteConversion types: ERROR ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
try {
|
||||
const siteEventTypes = await prisma.siteEvent.groupBy({
|
||||
by: ["eventType"],
|
||||
where: { brandId: brand.id },
|
||||
_count: true,
|
||||
});
|
||||
siteEventTypes.sort((a, b) => (b._count as unknown as number) - (a._count as unknown as number));
|
||||
log(`SiteEvent types (top 20): ${j(siteEventTypes.slice(0, 20))}`);
|
||||
log(`SiteEvent distinct type count: ${siteEventTypes.length}`);
|
||||
} catch (err) {
|
||||
log(`SiteEvent types: ERROR ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
// ─── 30-day windowed ground truth ──────────────────────────
|
||||
// The UI's default range is 30 days. Logging both all-time
|
||||
// and 30-day counts side by side so "why does the dashboard
|
||||
// show 528 conversions when the DB has 3000+?" is obvious.
|
||||
log("\n=== 30-day windowed counts ===");
|
||||
const since30d = new Date(Date.now() - 30 * 86_400_000);
|
||||
|
||||
try {
|
||||
const conv30 = await prisma.siteConversion.count({
|
||||
where: { brandId: brand.id, timestamp: { gte: since30d } },
|
||||
});
|
||||
log(`SiteConversion (30d): ${conv30}`);
|
||||
} catch (err) {
|
||||
log(`SiteConversion (30d): ERROR ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
if (site) {
|
||||
try {
|
||||
const te30 = await prisma.trackedEvent.count({
|
||||
where: { trackedSiteId: site.id, timestamp: { gte: since30d } },
|
||||
});
|
||||
log(`TrackedEvent (30d): ${te30}`);
|
||||
const sessStart30 = await prisma.trackedEvent.count({
|
||||
where: { trackedSiteId: site.id, eventType: "session_start", timestamp: { gte: since30d } },
|
||||
});
|
||||
log(`TrackedEvent session_start (30d): ${sessStart30}`);
|
||||
const pv30 = await prisma.trackedEvent.count({
|
||||
where: { trackedSiteId: site.id, eventType: "page_view", timestamp: { gte: since30d } },
|
||||
});
|
||||
log(`TrackedEvent page_view (30d): ${pv30}`);
|
||||
const ts30 = await prisma.trackedSession.count({
|
||||
where: { trackedSiteId: site.id, startedAt: { gte: since30d } },
|
||||
});
|
||||
log(`TrackedSession (30d): ${ts30}`);
|
||||
} catch (err) {
|
||||
log(`TrackedEvent/TrackedSession (30d): ERROR ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const gsc30 = await prisma.gscPage.aggregate({
|
||||
where: { brandId: brand.id, date: { gte: since30d } },
|
||||
_sum: { clicks: true, impressions: true },
|
||||
_avg: { position: true, ctr: true },
|
||||
});
|
||||
log(`GscPage (30d) aggregate: ${j(gsc30)}`);
|
||||
} catch (err) {
|
||||
log(`GscPage (30d): ERROR ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
try {
|
||||
const ai30 = await prisma.aiUsageLog.aggregate({
|
||||
where: { brandId: brand.id, timestamp: { gte: since30d } },
|
||||
_count: true,
|
||||
_sum: { estimatedCost: true, inputTokens: true, outputTokens: true },
|
||||
});
|
||||
log(`AiUsageLog (30d): ${j(ai30)}`);
|
||||
} catch (err) {
|
||||
log(`AiUsageLog (30d): ERROR ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
log("\n=== Audit complete ===");
|
||||
|
||||
// Flush the full run to scripts/audit-results.md so the result is
|
||||
// checked in alongside the code-side audit. Wrapping in ``` keeps
|
||||
// the markdown file renderable even for large JSON payloads.
|
||||
const outPath = "scripts/audit-results.md";
|
||||
const now = new Date().toISOString();
|
||||
const body = [
|
||||
"# Phase 1 — Metric Ground-Truth Audit",
|
||||
"",
|
||||
`**Generated:** ${now}`,
|
||||
`**Brand:** ${brand.name} (${brand.id})`,
|
||||
`**Domain:** ${brand.domain}`,
|
||||
"",
|
||||
"This file is produced by `scripts/audit-phase1.ts` and captures",
|
||||
"the raw per-table counts + event-type breakdowns for a single",
|
||||
"brand. These are the **ground-truth numbers** the UI surfaces",
|
||||
"must agree with.",
|
||||
"",
|
||||
"Re-run with `npx tsx scripts/audit-phase1.ts` — the file is",
|
||||
"overwritten in place.",
|
||||
"",
|
||||
"## Full output",
|
||||
"",
|
||||
"```text",
|
||||
...report,
|
||||
"```",
|
||||
"",
|
||||
].join("\n");
|
||||
mkdirSync(dirname(outPath), { recursive: true });
|
||||
writeFileSync(outPath, body, "utf8");
|
||||
console.log(`\n[audit-phase1] wrote ${outPath} (${report.length} lines)`);
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((err) => {
|
||||
console.error("[audit-phase1] FATAL", err);
|
||||
process.exit(1);
|
||||
})
|
||||
.finally(() => prisma.$disconnect());
|
||||
@@ -0,0 +1,273 @@
|
||||
# Phase 1 — Metric Ground-Truth Audit
|
||||
|
||||
**Status:** Script ready — needs a live `DATABASE_URL` to produce real numbers.
|
||||
|
||||
The `scripts/audit-phase1.ts` runner is a read-only Prisma script. Once
|
||||
it runs against production it overwrites this file with the actual
|
||||
per-table counts + event-type breakdowns for a single brand.
|
||||
|
||||
## How to run
|
||||
|
||||
```bash
|
||||
# Default: searches for a brand whose name contains "Modern Heart".
|
||||
DATABASE_URL=postgres://user:pass@host/db npx tsx scripts/audit-phase1.ts
|
||||
|
||||
# Override the brand:
|
||||
BRAND_NAME="Another Brand" DATABASE_URL=... npx tsx scripts/audit-phase1.ts
|
||||
```
|
||||
|
||||
On success the script overwrites this file with the ground-truth
|
||||
output from the target database.
|
||||
|
||||
## What the script captures
|
||||
|
||||
Scoped to a single brand, per table:
|
||||
|
||||
- **Brand-keyed tables** — `SiteConversion`, `SiteEvent`, `RealUserMetric`,
|
||||
`AiInteraction`, `AiUsageLog`, `PlatformEvent`, `MetricSnapshot`,
|
||||
`GscPage`, `TechnicalAudit`. For each: row count, min / max
|
||||
timestamp.
|
||||
- **TrackedSite-keyed tables** — `TrackedEvent`, `TrackedSession`. The
|
||||
script resolves the brand's `TrackedSite.id` once, then queries
|
||||
those two tables by `trackedSiteId`.
|
||||
- **Type breakdowns** — top 30 `TrackedEvent.eventType` buckets, top
|
||||
30 `SiteConversion.conversionType` buckets, top 20
|
||||
`SiteEvent.eventType` buckets.
|
||||
- **30-day windowed counts** — `SiteConversion`, `TrackedEvent`
|
||||
(total + `session_start` + `page_view`), `TrackedSession`,
|
||||
`GscPage` aggregate (clicks / impressions / position / ctr),
|
||||
`AiUsageLog` aggregate (cost / tokens).
|
||||
|
||||
Every query runs independently (no transaction) and errors surface as
|
||||
`ERROR <message>` lines rather than aborting the rest of the audit.
|
||||
|
||||
## Expected output shape
|
||||
|
||||
```text
|
||||
=== Brand ===
|
||||
{"brand":{"id":"cl...","name":"Modern Heart and Vascular","domain":"modernheartandvascular.com","createdAt":"2026-04-10T...Z"}}
|
||||
=== TrackedSite ===
|
||||
{"site":{"id":"cl...","status":"active","firstEventAt":"2026-04-10T...Z","lastEventAt":"2026-04-16T...Z","createdAt":"2026-04-10T...Z"}}
|
||||
|
||||
=== Brand-scoped tables ===
|
||||
SiteConversion: {"_count":NNN,"_min":{"timestamp":"..."},"_max":{"timestamp":"..."}}
|
||||
SiteEvent: {...}
|
||||
RealUserMetric: {...}
|
||||
AiInteraction: {...}
|
||||
AiUsageLog: {...}
|
||||
PlatformEvent: {...}
|
||||
MetricSnapshot: {...}
|
||||
GscPage: {...}
|
||||
TechnicalAudit: {...}
|
||||
|
||||
=== TrackedSite-scoped tables ===
|
||||
TrackedEvent: {"_count":NNN,"_min":{"timestamp":"..."},"_max":{"timestamp":"..."}}
|
||||
TrackedSession: {...}
|
||||
|
||||
=== Distinct event-type counts ===
|
||||
TrackedEvent types (top 30): [{"eventType":"page_view","_count":N},...]
|
||||
TrackedEvent distinct type count: NN
|
||||
SiteConversion types (top 30): [{"conversionType":"form_submit","_count":N},...]
|
||||
SiteConversion distinct type count: NN
|
||||
SiteEvent types (top 20): [{"eventType":"outbound_click","_count":N},...]
|
||||
SiteEvent distinct type count: NN
|
||||
|
||||
=== 30-day windowed counts ===
|
||||
SiteConversion (30d): NNN
|
||||
TrackedEvent (30d): NNN
|
||||
TrackedEvent session_start (30d): NNN
|
||||
TrackedEvent page_view (30d): NNN
|
||||
TrackedSession (30d): NNN
|
||||
GscPage (30d) aggregate: {"_sum":{"clicks":N,"impressions":N},"_avg":{"position":N,"ctr":N}}
|
||||
AiUsageLog (30d): {"_count":N,"_sum":{"estimatedCost":N,"inputTokens":N,"outputTokens":N}}
|
||||
|
||||
=== Audit complete ===
|
||||
```
|
||||
|
||||
## Why these tables
|
||||
|
||||
Every metric surfaced on the app dashboard, admin dashboard, and
|
||||
Site Tag Analytics pages ultimately reads from one of these tables.
|
||||
Grounding the audit here lets us confirm:
|
||||
|
||||
- Whether a "528 conversions" UI display actually matches
|
||||
`SELECT COUNT(*) FROM "SiteConversion" WHERE brandId=...
|
||||
AND timestamp>=<30d>` on the live DB.
|
||||
- Whether `TrackedEvent.session_start` and `TrackedSession` diverge
|
||||
(the root cause of the 16,382 vs 16,447 session drift the audit
|
||||
already flagged in code).
|
||||
- Which `eventType` values actually populate the DB — required for
|
||||
deciding which buckets each funnel stage rolls up.
|
||||
|
||||
## Next phases
|
||||
|
||||
Once the script runs and populates this file, the numbers printed
|
||||
here become the "truth" column against which every API response and
|
||||
UI card is compared. Phases 2+ replace ad-hoc per-surface queries
|
||||
with shared utilities that match these ground-truth numbers.
|
||||
|
||||
---
|
||||
|
||||
## Sessions
|
||||
|
||||
Every call site in `src/` that queries or displays a session count.
|
||||
Format: `path:line` — source table — filter — dedup status — scope.
|
||||
|
||||
### TrackedSession (deduped — the authoritative unique-visitor count)
|
||||
|
||||
- `src/app/api/admin/platform-metrics/route.ts:203` — `prisma.trackedSession.findMany({ where: sessionWhere(start, end), select: { startedAt: true }, take: 100_000 })` — **TrackedSession** — range + trackedSiteId — deduped by construction — single brand OR all brands.
|
||||
- `src/app/api/admin/platform-metrics/route.ts:208` — `prisma.trackedSession.count({ where: sessionWhere(prevStart, prevEnd) })` — **TrackedSession** — previous range — deduped — same scope as above (drives the trend %).
|
||||
- `src/app/api/admin/platform-metrics/route.ts:389` — `prisma.trackedSession.groupBy({ by: ["trackedSiteId"], where: { startedAt: { gte: start, lt: end } }, _count: true, orderBy: { _count: { trackedSiteId: "desc" } }, take: 20 })` — **TrackedSession** — window only — deduped — cross-brand Top Brands list.
|
||||
- `src/app/api/admin/platform-metrics/route.ts:628` — `prisma.trackedSession.count({ where: { ...trackedSiteIdFilter, startedAt: { gte: compStart, lte: compEnd } } })` — **TrackedSession** — overlap window — deduped — used for Site Tag vs GA4 reconciliation.
|
||||
- `src/app/api/admin/sessions/route.ts:148` — `prisma.trackedSession.count({ where: sessionWhere })` — **TrackedSession** — brand + optional date range — deduped — powers the admin Session Explorer list.
|
||||
- `src/app/api/admin/brands/[brandId]/route.ts:59` — `prisma.trackedSession.count({ where: { trackedSiteId } })` — **TrackedSession** — no window — deduped — all-time count on the admin brand detail page.
|
||||
- `src/app/api/admin/analytics/data-moat/route.ts:48` — `prisma.trackedSession.count()` — **TrackedSession** — no filter — all-time platform total.
|
||||
- `src/app/api/admin/analytics/data-moat/route.ts:62` — `prisma.trackedSession.count({ where: { startedAt: { gte: sevenDaysAgo } } })` — **TrackedSession** — 7d — for "growth per day" rate.
|
||||
- `src/app/api/admin/analytics/data-moat/route.ts:152` — `prisma.trackedSession.groupBy({ by: ["trackedSiteId"], _count: true })` — **TrackedSession** — all-time — powers the per-brand data-moat leaderboard.
|
||||
- `src/app/api/admin/analytics/funnel/route.ts:133` — `prisma.trackedSession.findMany({ where: sessionWhere, select: { id, sessionId, startedAt, pageCount } })` — **TrackedSession** — per-brand window — drives the 5-stage funnel denominator.
|
||||
- `src/app/api/admin/analytics/growth/route.ts:71` — `prisma.trackedSession.findMany({ where: { startedAt: { gte: start } }, select: { startedAt: true } })` — **TrackedSession** — monthly rollup — platform growth chart.
|
||||
- `src/app/api/admin/export/platform-metrics/route.ts:55` — `prisma.trackedSession.count()` — **TrackedSession** — all-time — export row "totals.sessions".
|
||||
- `src/app/api/admin/export/growth-data/route.ts:58` — `prisma.trackedSession.count({ where: { startedAt: { gte: m.start, lt: m.end } } })` — **TrackedSession** — per-month — export growth csv.
|
||||
- `src/app/api/admin/export/brand-data/route.ts:47` — `prisma.trackedSession.groupBy({ by: ["trackedSiteId"], where: { trackedSiteId: { in: trackedSiteIds } }, _count: true })` — **TrackedSession** — per-brand — brand export csv.
|
||||
- `src/app/api/admin/system-health/route.ts:45` — `prisma.trackedSession.count()` — **TrackedSession** — all-time — system-health volume row.
|
||||
- `src/app/api/content-audit/page-detail/route.ts:158` — `prisma.trackedSession.count({ where: { trackedSiteId, startedAt: { gte: since } } })` — **TrackedSession** — 30d — per-page conversion-rate denominator.
|
||||
- `src/app/api/content-audit/page-detail/route.ts:161` — `prisma.trackedSession.count({ where: { trackedSiteId, startedAt: { gte: prevSince, lt: since } } })` — **TrackedSession** — previous 30d — delta.
|
||||
- `src/app/api/content-briefs/insights/route.ts:267` — `prisma.trackedSession.groupBy({ by: ["country"], where: { trackedSiteId, country: { not: null } }, _count: true, orderBy: { _count: { country: "desc" } }, take: 5 })` — **TrackedSession** — all-time — top visitor countries (relabelled `sessions: g._count`).
|
||||
- `src/app/api/content-briefs/brand-profile/route.ts:147` — same as above — duplicated in brand-profile builder.
|
||||
- `src/app/api/cron/signals-site-tag-ai/route.ts:174` — `prisma.trackedSession.count({ where: { trackedSiteId, startedAt: { gte: twentyFourHoursAgo } } })` — **TrackedSession** — 24h — signal-detection cron.
|
||||
- `src/app/api/site-tag/confidence/route.ts:220` — `prisma.trackedSession.count({ where: { trackedSiteId, startedAt: { gte: effectiveFrom } } })` — **TrackedSession** — effective-overlap window — powers the Data Confidence "Session match rate" factor.
|
||||
- `src/app/api/site-tag/route.ts:94` — `prisma.trackedSession.count({ where: { trackedSiteId } })` — **TrackedSession** — all-time — snippet status card.
|
||||
- `src/lib/services/site-tag-analytics.ts:523` — `prisma.trackedSession.count({ where: { trackedSiteId, startedAt: { gte: from } } })` — **TrackedSession** — range — `getTrafficSummary()`, the Overview tab's Sessions KPI source.
|
||||
- `src/lib/services/site-tag-analytics.ts:661` — `prisma.trackedSession.count({ where: { trackedSiteId, startedAt: { gte: from } } })` — **TrackedSession** — range — `getConversionSummary()`, the Conversion Rate denominator.
|
||||
- `src/lib/services/brand-radar.ts:113` — `prisma.trackedSession.count({ where: { trackedSiteId, startedAt: { gte: d30 } } })` — **TrackedSession** — 30d — brand radar KPI.
|
||||
- `src/lib/services/brand-radar.ts:116` — `prisma.trackedSession.aggregate({ ... _avg: { pageCount } ... })` — **TrackedSession** — pages-per-session aggregate.
|
||||
- `src/lib/services/brand-radar.ts:396` — `prisma.trackedSession.count({ where: { trackedSiteId, startedAt: { gte: d7 } } })` — **TrackedSession** — 7d — brand radar delta.
|
||||
|
||||
### TrackedEvent.session_start (RAW event rows — NOT deduped)
|
||||
|
||||
- `src/app/api/ai-strategist/chat/route.ts:453` — inside the Site Tag metrics block, `session_start` appears in the `EVENT_TYPES` list that's fed to `trackedEvent.groupBy({ by: ["eventType"] })` — **TrackedEvent** — 28d — raw counts powering the `<site-tag>` context injected into the Strategist prompt.
|
||||
- `src/app/api/ai-strategist/chat/route.ts:490` — `const sessions = counts["session_start"] ?? 0` — derives the session count from the groupBy bucket — **TrackedEvent** — 28d — raw.
|
||||
- `src/app/api/ai-strategist/chat/route.ts:682` — `where: { trackedSiteId, eventType: "session_start", timestamp: { gte: start } }` — inside `buildDailyConversionContext()` for the daily conversion block — **TrackedEvent** — 28d — raw.
|
||||
- `src/app/api/ai-strategist/chat/route.ts:1032` — `prisma.trackedEvent.count({ where: { trackedSiteId, eventType: "session_start", timestamp: { gte: since } } })` — **TrackedEvent** — 28d — capture-rate comparison against GA4 in `buildTrackingHealth()`.
|
||||
- `src/lib/services/conversion-metrics.ts:145` — `prisma.trackedEvent.count({ where: { trackedSiteId, eventType: "session_start", timestamp: { gte: since, lte: now } } })` — **TrackedEvent** — param-driven — raw — the shared `getConversionMetrics()` helper's session denominator.
|
||||
- `src/lib/services/signal-detection.ts:380` — `prisma.trackedEvent.count({ where: { trackedSiteId, eventType: "session_start", timestamp: { gte: weekAgo } } })` — **TrackedEvent** — 7d — raw.
|
||||
- `src/lib/services/signal-detection.ts:383` — `prisma.trackedEvent.count({ where: { trackedSiteId, eventType: "session_start", timestamp: { gte: twoWeeksAgo, lt: weekAgo } } })` — **TrackedEvent** — previous 7d — raw.
|
||||
- `src/lib/services/signal-detection.ts:387` — `prisma.trackedEvent.count({ where: { trackedSiteId, eventType: "session_start", timestamp: { gte: twoDaysAgo } } })` — **TrackedEvent** — 48h — raw.
|
||||
- `src/lib/services/email-report-generator.ts:116` — `prisma.trackedEvent.count({ where: { trackedSiteId, eventType: "session_start", timestamp: { gte: since } } })` — **TrackedEvent** — per report window — raw.
|
||||
|
||||
### GA4 sessions (external API / MetricSnapshot — not Site Tag)
|
||||
|
||||
- `src/app/api/dashboard/route.ts:166, 276, 304, 339, 386` — `fetchGa4` + `fetchGa4StaleFallback` — GA4 Data API live call + `prisma.metricSnapshot.aggregate({ ..., _sum: { sessions } })` fallback — **external + MetricSnapshot** — dashboard GA4 card.
|
||||
- `src/app/api/dashboard/route.ts:932-1031` — merges live vs cached GA4 sessions into the dashboard response — **mixed source**.
|
||||
- `src/app/api/ai-strategist/chat/route.ts:866` — `prisma.metricSnapshot.aggregate({ where: { brandId, source: "ga4", date: { gte: since } }, _sum: { sessions: true } })` — **MetricSnapshot** — 28d — used by `buildTrackingHealth()` to derive a GA4 session count for the capture-rate comparison.
|
||||
|
||||
### UI-layer displays (no direct query — rebuild from prop shape)
|
||||
|
||||
- `src/app/site-tag-analytics/overview-tab.tsx:192` — `const tagSessions = traffic?.sessions ?? 0` — reads the `traffic.sessions` field of the `/api/site-tag/analytics` response (backed by `getTrafficSummary`, TrackedSession).
|
||||
- `src/app/site-tag-analytics/funnel-tab.tsx:70` — `const sessions = traffic.sessions` — same prop chain.
|
||||
- `src/app/admin/page.tsx` + `src/app/admin/analytics/data-moat/page.tsx:123` — render `trackedSessions` volume card from `/api/admin/analytics/data-moat` (TrackedSession count).
|
||||
- `src/app/admin/sessions/page.tsx:317` — renders count from `/api/admin/sessions` (TrackedSession).
|
||||
|
||||
### Drift analysis
|
||||
|
||||
**Core split.** Two tables answer "how many sessions did this brand have?":
|
||||
|
||||
1. **TrackedSession** — one row per browser session, keyed by the `sessionId` cookie set on the first `session_start`. The `lastSeenAt` column is touched on every subsequent event, but the row itself is unique per session.
|
||||
2. **TrackedEvent.session_start** — a TrackedEvent row is written every time `t.js` emits a `session_start` event (typically once per 30-minute idle-timeout window). If a user returns after the 30-minute timeout, `t.js` re-emits `session_start` and a NEW TrackedSession row IS created too, so the two counts normally track closely. Divergence sources:
|
||||
|
||||
- **Clock-skew / race writes.** Ingest writes TrackedEvent first, then conditionally upserts TrackedSession (see `recordEvent` in `src/lib/services/site-tag.ts`). If the TrackedSession upsert fails silently, TrackedEvent has a `session_start` row without a matching TrackedSession row → TrackedEvent count > TrackedSession count.
|
||||
- **Historical data without sessionId.** Older TrackedEvent rows predate the `sessionId` column — those `session_start` rows get counted in the event query but never had a TrackedSession row.
|
||||
- **TrackedSession backfill gaps.** The `TrackedSession.startedAt` column defaults to `now()` on insert but the TrackedEvent `timestamp` is set from the client payload. On a long upload (sendBeacon queued during an outage), the TrackedEvent lands in the prior day but the TrackedSession lands in the recovery day — they don't align across window edges.
|
||||
|
||||
Net: for a typical brand TrackedSession is the smaller / truer count and TrackedEvent `session_start` overcounts slightly.
|
||||
|
||||
**The one surface that drifts today.** Every admin + app surface uses **TrackedSession** for the top-line Session count (overview, admin dashboard, content-audit, brand-radar, confidence). The exception is **`src/lib/services/conversion-metrics.ts:145`** — the shared `getConversionMetrics()` helper counts `TrackedEvent.session_start` instead of `TrackedSession`. Any surface calling the helper for its Conversion Rate denominator will report a slightly inflated session count + slightly deflated rate vs surfaces that call `getConversionSummary()` / `getTrafficSummary()` directly.
|
||||
|
||||
Action item for Phase 4: swap `getConversionMetrics` over to `prisma.trackedSession.count({ where: { trackedSiteId, startedAt: { gte: since } } })` so every surface agrees on the denominator.
|
||||
|
||||
**Report-only sites.** `ai-strategist/chat` + `signal-detection` + `email-report-generator` also use `TrackedEvent.session_start`. These are narrative contexts (AI prompt, signal digest, weekly email) where the ±3% noise doesn't change the story, but they should migrate for consistency once the shared helper lands.
|
||||
|
||||
**GA4 vs Site Tag.** Separately, the dashboard's GA4 Sessions card reads from the GA4 Data API (or MetricSnapshot fallback). That number is **not** comparable to the Site Tag session count — it's a different measurement system (cookies + Google's sampling) and a different population (no ad-blocker coverage). The Data Confidence factor uses the **effective overlap window** (install date → now) to compare apples-to-apples; every other surface must not compute a %-difference between the two.
|
||||
|
||||
**Range boundary.** Admin uses `prevStart, prevEnd` for comparison; `/api/site-tag/analytics` uses `rangeStart()`; data-moat hardcodes `sevenDaysAgo` / `30 days`. These boundaries are all correct for their use but produce different numbers for the "same" 30-day view — labels must make the window explicit (handled in Phase 6).
|
||||
|
||||
---
|
||||
|
||||
## Shared Utility Migration Status
|
||||
|
||||
**Completed:** 2026-04-17
|
||||
|
||||
All user-visible brand-scoped metric queries now route through
|
||||
`src/lib/services/platform-metrics.ts`. The remaining inline queries
|
||||
are categorised below — every one has been reviewed and has a
|
||||
documented reason for staying inline.
|
||||
|
||||
### Migrated to shared helpers (brand-scoped KPIs)
|
||||
|
||||
| Surface | Metric | Shared helper |
|
||||
|---|---|---|
|
||||
| site-tag-analytics getTrafficSummary | Sessions, Page Views | `getSessionCount`, `getPageViewCount` |
|
||||
| site-tag-analytics getConversionSummary | Session denominator | `getSessionCount` |
|
||||
| site-tag-analytics detectTrackedSignals | Phone calls | `getPhoneMetrics` |
|
||||
| site-tag/analytics/ux route | Sessions, UX signals, Form submits, Phone, Bookings, CWV | `getSessionCount`, `getUXSignals`, `getConversionMetrics` |
|
||||
| site-tag/analytics/rum route | CWV | P75 + outlier filter (aligned with `getCoreWebVitals`) |
|
||||
| site-tag/confidence route | Sessions, Install date | `getSessionCount`, `getSiteTagInstalledAt` |
|
||||
| admin/platform-metrics route | Form submits, Bookings, Phone, Comparison sessions | `dedupedConversionMetric(formSubmitWhere/bookingWhere/phoneWhere)`, `getSessionCount` |
|
||||
| admin/conversion-reconciliation | Form starts | `getFormMetrics` |
|
||||
| ai-strategist/chat buildSiteTagContext | Sessions | `getSessionCount` |
|
||||
| ai-strategist/chat buildTrackingHealth | Sessions, GSC clicks | `getSessionCount`, `getGSCMetrics` |
|
||||
| dashboard route | GSC DB fallbacks (×4) | `getGSCMetrics` |
|
||||
| brand-radar service | Sessions (30d + 7d) | `getSessionCount` |
|
||||
| email-report-generator service | Sessions, Page Views | `getSessionCount`, `getPageViewCount` |
|
||||
| analysis-data-package service | Sessions | `getSessionCount` |
|
||||
|
||||
### Remaining inline — cross-brand admin (no brandId)
|
||||
|
||||
These are platform-wide aggregations that don't take a brandId —
|
||||
the shared helpers are brand-scoped by design.
|
||||
|
||||
- `admin/system-health` — table-volume counts (all brands)
|
||||
- `admin/analytics/data-moat` — platform-wide totals + per-brand groupBys
|
||||
- `admin/analytics/growth` — monthly growth timeseries
|
||||
- `admin/export/growth-data` — monthly CSV export
|
||||
- `admin/export/brand-data` — cross-brand groupBy
|
||||
- `admin/cost-center` — platform-wide event + session + conversion totals
|
||||
- `cron/platform-snapshot` — daily cron snapshot
|
||||
|
||||
### Remaining inline — domain-specific (not standard KPIs)
|
||||
|
||||
- `admin/sessions/route` — session list pagination count
|
||||
- `admin/brands/[brandId]` — all-time counts for brand detail card
|
||||
- `admin/conversion-reconciliation` — per-canonical-type SiteConversion groupBy
|
||||
- `admin/fix-attribution` — admin debug tool
|
||||
- `content-audit/page-detail` — per-page session/event/conversion counts
|
||||
- `site-tag/route` — tag status + snippet config
|
||||
- `site-tag/analytics/route` — lifetime event count for Data Quality
|
||||
- `site-tag/analytics/pulse` — today's event count for live pulse
|
||||
- `ai-strategist/chat` — SiteConversion total + groupBy for AI context
|
||||
- `paid-media/*` — domain-specific attribution queries
|
||||
- `cron/signals-site-tag-ai` — per-brand signal detection
|
||||
|
||||
### Remaining inline — service files
|
||||
|
||||
- `brand-radar` — raw event volume + SiteConversion count (total metrics)
|
||||
- `email-report-generator` — form_start count (intent signal)
|
||||
- `billing` — TrackedEvent count for plan usage metering
|
||||
- `benchmark-engine` — cross-brand conversion count
|
||||
- `industry-intel` — cross-brand conversion count
|
||||
- `site-tag-status` — tag health check
|
||||
- `notification-service` — current vs previous conversion counts for alerts
|
||||
- `signal-detection` — week-over-week conversion + session comparison
|
||||
|
||||
### SiteEvent safety audit
|
||||
|
||||
Every SiteEvent query falls into one of these categories:
|
||||
|
||||
- **Filtered by eventType** — signals, errors, UX, heatmap, outbound (safe)
|
||||
- **Filtered by NOT NOISE_SITE_EVENT_TYPES** — system-health, brands, export, snapshot, overview (safe)
|
||||
- **Filtered by ACTIONABLE_TYPES** — admin/signals (safe)
|
||||
- **Session/ID bounded** — compliance export/delete (safe)
|
||||
- **Write operations** — competitor-crawl, industry-intel creates (safe)
|
||||
- **Retention purge** — data-retention findMany + deleteMany (intentional)
|
||||
|
||||
No unfiltered-read SiteEvent queries remain.
|
||||
@@ -0,0 +1,374 @@
|
||||
# Brand Scoping Audit
|
||||
|
||||
> Generated 2026-04-17. Every user-facing page, API route, and service
|
||||
> file was checked to confirm it scopes data by `brandId` so new brands
|
||||
> work out of the box and no page leaks another brand's data.
|
||||
|
||||
## Overall Assessment: WELL SCOPED
|
||||
|
||||
The codebase has strong brand isolation across all surfaces.
|
||||
|
||||
### Deep Query Audit (2026-04-17)
|
||||
|
||||
Every critical API route was verified to use brandId in ALL Prisma
|
||||
queries — no cross-brand data leakage detected. Specific findings:
|
||||
|
||||
- All routes validate brandId presence (400 or empty response if missing)
|
||||
- All routes verify org membership before returning data
|
||||
- All TrackedEvent queries use `{ trackedSite: { brandId } }` relation
|
||||
filter (TrackedEvent has no direct brandId column)
|
||||
- `/api/notifications` returns `[]` when brandId is missing (prevents
|
||||
infinite re-render in topnav bell)
|
||||
- `/api/technical-audit` returns 400 when brandId is missing (page guards
|
||||
fetches until brand context resolves)
|
||||
|
||||
---
|
||||
|
||||
## 1. PROPERLY SCOPED — App Pages
|
||||
|
||||
All 20+ app pages use `useBrand()` context to read the active brand
|
||||
and pass `selectedBrand.id` to every API call.
|
||||
|
||||
| Page | Brand Context |
|
||||
|------|---------------|
|
||||
| `src/app/dashboard/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/site-tag-analytics/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/site-tag/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/execution-hub/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/competitor-intel/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/integrations/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/broken-links/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/content-hub/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/page-launch-tracker/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/ai-visibility/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/rank-tracker/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/keyword-research/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/report-studio/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/site-performance-audit/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/brands-hub/page.tsx` | `useBrand()` → `orgId` |
|
||||
| `src/app/paid-media/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/settings/page.tsx` | `useBrand()` context |
|
||||
| `src/app/schema-markup/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/content-writing/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
| `src/app/ai-strategist/page.tsx` | `useBrand()` → `selectedBrand.id` |
|
||||
|
||||
Pages that don't use `useBrand()` (correctly — they don't display brand data):
|
||||
|
||||
| Page | Reason |
|
||||
|------|--------|
|
||||
| `src/app/website-upload/page.tsx` | Brand creation flow — no existing data |
|
||||
| `src/app/pricing/page.tsx` | Static pricing page |
|
||||
| `src/app/preview/page.tsx` | Gets `brandId` from URL params (opened by broken-links page) |
|
||||
| `src/app/client-portal/[token]/page.tsx` | Token-authenticated — resolves brand from token |
|
||||
|
||||
## 2. PROPERLY SCOPED — API Routes
|
||||
|
||||
All 49+ non-admin API routes accept and filter by `brandId` (or resolve
|
||||
via `siteId` for Site Tag ingestion routes).
|
||||
|
||||
| Route | brandId Source |
|
||||
|-------|---------------|
|
||||
| `src/app/api/dashboard/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/site-tag/analytics/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/site-tag/analytics/ux/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/site-tag/analytics/errors/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/site-tag/analytics/video-chat/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/site-tag/analytics/rum/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/site-tag/confidence/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/site-tag/competitors/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/site-tag/page-links/route.ts` | Query param `brandId` or `siteId` |
|
||||
| `src/app/api/site-tag/event/route.ts` | Body `siteId` → resolves brandId |
|
||||
| `src/app/api/site-tag/rum/route.ts` | Body `siteId` → resolves brandId |
|
||||
| `src/app/api/gsc/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/gsc/backfill/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/competitors/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/competitors/discover/route.ts` | Body `brandId` |
|
||||
| `src/app/api/technical-audit/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/content-brief/route.ts` | Query param/body `brandId` |
|
||||
| `src/app/api/content-brief/generate/route.ts` | Body `brandId` |
|
||||
| `src/app/api/content-writing/route.ts` | Query param/body `brandId` |
|
||||
| `src/app/api/ai-strategist/chat/route.ts` | Body `brandId` |
|
||||
| `src/app/api/dead-pages/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/dead-pages/run/route.ts` | Body `brandId` |
|
||||
| `src/app/api/page-launch/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/rank-tracker/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/rank-tracker/check/route.ts` | Body `brandId` |
|
||||
| `src/app/api/brands/route.ts` | Query param `orgId` |
|
||||
| `src/app/api/brands/[brandId]/route.ts` | URL param `brandId` |
|
||||
| `src/app/api/brands/[brandId]/assets/route.ts` | URL param `brandId` |
|
||||
| `src/app/api/brands/[brandId]/assets/[assetId]/route.ts` | URL param `brandId` |
|
||||
| `src/app/api/brands/[brandId]/has-site-tag-data/route.ts` | URL param `brandId` |
|
||||
| `src/app/api/brands/[brandId]/enrichment-files/route.ts` | URL param `brandId` |
|
||||
| `src/app/api/brands/[brandId]/enrichment-files/[fileId]/route.ts` | URL param `brandId` |
|
||||
| `src/app/api/brands/extract/route.ts` | Brand discovery — no existing data |
|
||||
| `src/app/api/integrations/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/integrations/disconnect/route.ts` | Body `brandId` |
|
||||
| `src/app/api/keyword-research/route.ts` | Query param/body `brandId` |
|
||||
| `src/app/api/keyword-research/saved/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/keyword-research/[id]/route.ts` | Loads by ID, verifies brand access |
|
||||
| `src/app/api/me/plan/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/notifications/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/report-studio/generate/route.ts` | Body `brandId` |
|
||||
| `src/app/api/report-studio/download/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/report-studio/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/schema/generate/route.ts` | Body `brandId` |
|
||||
| `src/app/api/paid-media/overview/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/paid-media/attribution/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/paid-media/campaigns/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/paid-media/visitors/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/paid-media/households/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/paid-media/reconciliation/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/ai-visibility/route.ts` | Query param `brandId` |
|
||||
| `src/app/api/email-reports/route.ts` | Query param/body `brandId` |
|
||||
|
||||
## 3. PROPERLY SCOPED — Service Files
|
||||
|
||||
All service files accept `brandId` as a parameter and use it in queries.
|
||||
|
||||
| Service | Key Functions |
|
||||
|---------|---------------|
|
||||
| `src/lib/services/platform-metrics.ts` | All functions take `(brandId, since)` |
|
||||
| `src/lib/services/conversion-metrics.ts` | `getConversionMetrics(brandId, since)` |
|
||||
| `src/lib/services/paid-media-metrics.ts` | All functions take `brandId` |
|
||||
| `src/lib/services/attribution.ts` | `buildAttributionPaths(brandId)` |
|
||||
| `src/lib/services/site-tag-analytics.ts` | `getSiteTagAnalytics(brandId, range)` |
|
||||
| `src/lib/services/site-tag.ts` | `recordEvent(siteId, ...)` — scoped by site |
|
||||
| `src/lib/services/audit-runner.ts` | `runTechnicalAudit(brandId, ...)` |
|
||||
| `src/lib/services/signal-detection.ts` | Functions take `brandId` |
|
||||
| `src/lib/services/auto-backfill.ts` | `autoBackfillConversions(brandId)` |
|
||||
| `src/lib/services/visitor-profiles.ts` | All take `brandId` |
|
||||
| `src/lib/services/ott-attribution.ts` | All take `brandId` |
|
||||
| `src/lib/services/email-report-generator.ts` | Takes `brandId` |
|
||||
| `src/lib/services/ai-analysis-runner.ts` | Takes `brandId` |
|
||||
| `src/lib/services/analysis-data-package.ts` | Takes `brandId` |
|
||||
| `src/lib/services/brand-radar.ts` | Takes `brandId` |
|
||||
| `src/lib/services/cta-optimizer.ts` | Takes `brandId` |
|
||||
| `src/lib/services/keyword-suggestions.ts` | Takes `brandId` |
|
||||
| `src/lib/services/page-build-context.ts` | Takes `brandId` |
|
||||
| `src/lib/services/paid-media-context.ts` | Takes `brandId` |
|
||||
| `src/lib/services/dead-page-scanner.ts` | Takes `brandId` |
|
||||
| `src/lib/services/integration-credentials.ts` | `getCredentials(brandId, integrationId)` |
|
||||
| `src/lib/services/platform-telemetry.ts` | Takes `brandId` (optional) |
|
||||
| `src/lib/services/page-lifecycle.ts` | Functions resolve via `siteId` → `brandId` |
|
||||
|
||||
## 4. MISSING BRAND SCOPE — Security Issues
|
||||
|
||||
### 4a. No Brand Ownership Verification (CRITICAL)
|
||||
|
||||
These routes authenticate the user but don't verify the target resource
|
||||
belongs to a brand in the user's organization. A user could read, modify,
|
||||
or delete another org's data by guessing cuid IDs.
|
||||
|
||||
| Priority | File | Issue | Fix |
|
||||
|----------|------|-------|-----|
|
||||
| CRITICAL | `src/app/api/schema/[id]/route.ts` | PUT/DELETE update or delete any schema by ID without checking brand ownership. | Load schema with `include: { brand: { include: { organization: { include: { memberships: true } } } } }`, verify `userId` in memberships. |
|
||||
| CRITICAL | `src/app/api/schema/save/route.ts` | POST creates schema for any `brandId` without verifying org access. | Verify the caller belongs to the brand's org before creating. |
|
||||
| CRITICAL | `src/app/api/content-briefs/export/route.ts` | GET exports any brief by ID without brand ownership check. | Load brief with brand → org → memberships, verify access. |
|
||||
| CRITICAL | `src/app/api/content-briefs/links/route.ts` | POST regenerates links for any brief by ID without ownership check. | Same pattern. |
|
||||
| CRITICAL | `src/app/api/content-briefs/score/route.ts` | GET/POST read and recalculate scores for any brief by ID. | Same pattern. |
|
||||
| CRITICAL | `src/app/api/writing-assistant/documents/[id]/route.ts` | GET/DELETE read or delete any writing document by ID. | Load doc with brand → org → memberships, verify access. |
|
||||
| HIGH | `src/app/api/tasks/[id]/stage/route.ts` | PUT updates any task's pipeline stage by ID. | Load task with brand → org → memberships, verify access. |
|
||||
| MEDIUM | `src/app/api/schema/validate/route.ts` | POST has no auth at all — completely public. | Add `getCurrentUser()` check. Validation is stateless so risk is limited. |
|
||||
|
||||
### 4b. Missing Brand Filter (LOW)
|
||||
|
||||
| Priority | File | Issue | Fix |
|
||||
|----------|------|-------|-----|
|
||||
| LOW | `src/app/api/ai-strategist/sessions/route.ts` | GET fetches strategist sessions filtered by `userId` only, not by `brandId`. Sessions from all brands appear together. | Add optional `brandId` query param filter. |
|
||||
|
||||
## 5. ADMIN-ONLY — Intentionally Cross-Brand
|
||||
|
||||
### Admin Pages
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `src/app/admin/page.tsx` | Admin dashboard overview |
|
||||
| `src/app/admin/brands/page.tsx` | All brands management |
|
||||
| `src/app/admin/sessions/page.tsx` | Session explorer |
|
||||
| `src/app/admin/sessions/[sessionId]/page.tsx` | Session detail |
|
||||
| `src/app/admin/historical/page.tsx` | Historical tracking |
|
||||
| `src/app/admin/ai-usage/page.tsx` | AI usage monitoring |
|
||||
| `src/app/admin/activity/page.tsx` | Platform activity |
|
||||
| `src/app/admin/integrations/page.tsx` | Integration status |
|
||||
| `src/app/admin/analytics/page.tsx` | Analytics hub |
|
||||
| `src/app/admin/analytics/funnel/page.tsx` | Growth funnel |
|
||||
| `src/app/admin/paid-media/page.tsx` | Paid media admin |
|
||||
| `src/app/admin/compliance/page.tsx` | Compliance dashboard |
|
||||
| `src/app/admin/clients/page.tsx` | Client management |
|
||||
| `src/app/admin/cost-center/page.tsx` | Cost center |
|
||||
| `src/app/admin/data-quality/page.tsx` | Data quality |
|
||||
|
||||
### Admin API Routes
|
||||
|
||||
All gated by `requireSuperadmin()`.
|
||||
|
||||
| Route | Purpose |
|
||||
|-------|---------|
|
||||
| `src/app/api/admin/overview/route.ts` | Dashboard stats |
|
||||
| `src/app/api/admin/brands/route.ts` | Brand list |
|
||||
| `src/app/api/admin/brands/[brandId]/route.ts` | Brand detail/delete |
|
||||
| `src/app/api/admin/brands/[brandId]/transfer/route.ts` | Transfer brand |
|
||||
| `src/app/api/admin/sessions/route.ts` | Session list |
|
||||
| `src/app/api/admin/sessions/[sessionId]/route.ts` | Session detail |
|
||||
| `src/app/api/admin/signals/route.ts` | Signal detection |
|
||||
| `src/app/api/admin/integrations/route.ts` | Integration status |
|
||||
| `src/app/api/admin/system-health/route.ts` | System health |
|
||||
| `src/app/api/admin/historical/route.ts` | Historical data |
|
||||
| `src/app/api/admin/historical/export/route.ts` | CSV export |
|
||||
| `src/app/api/admin/analytics/data-moat/route.ts` | Data moat metrics |
|
||||
| `src/app/api/admin/analytics/funnel/route.ts` | Growth funnel |
|
||||
| `src/app/api/admin/cost-center/route.ts` | Cost tracking |
|
||||
| `src/app/api/admin/compliance/overview/route.ts` | Compliance |
|
||||
| `src/app/api/admin/platform-metrics/route.ts` | Platform metrics |
|
||||
| `src/app/api/admin/export/platform-metrics/route.ts` | Metrics export |
|
||||
| `src/app/api/admin/conversion-reconciliation/route.ts` | Reconciliation |
|
||||
| `src/app/api/admin/paid-media/route.ts` | Paid media admin |
|
||||
| `src/app/api/admin/debug/backfill-conversions/route.ts` | Debug backfill |
|
||||
| `src/app/api/admin/debug/backfill-gsc-history/route.ts` | GSC backfill |
|
||||
| `src/app/api/admin/debug/backfill-ga4-history/route.ts` | GA4 backfill |
|
||||
| `src/app/api/admin/debug/backfill-snapshots/route.ts` | Snapshot backfill |
|
||||
| `src/app/api/admin/debug/verify-events/route.ts` | Event verification |
|
||||
|
||||
## 6. INFRASTRUCTURE — No Brand Scoping Needed
|
||||
|
||||
| Route | Purpose |
|
||||
|-------|---------|
|
||||
| `src/app/api/collect/route.ts` | Site Tag event collection (uses siteId) |
|
||||
| `src/app/api/site-tag/event/route.ts` | Site event recording (uses siteId) |
|
||||
| `src/app/api/site-tag/conversion/route.ts` | Conversion recording (uses siteId) |
|
||||
| `src/app/api/site-tag/script/[name]/route.ts` | Static JS module serving |
|
||||
| `src/app/api/cron/data-retention/route.ts` | Data retention cleanup |
|
||||
| `src/app/api/cron/platform-snapshot/route.ts` | Daily snapshots |
|
||||
| `src/app/api/cron/paid-media-rebuild/route.ts` | Paid media rebuild |
|
||||
| `src/app/api/onboarding/route.ts` | Brand creation |
|
||||
| `src/app/api/upload/route.ts` | File upload |
|
||||
| `src/app/api/track/route.ts` | Telemetry ingestion |
|
||||
| `src/app/api/organizations/members/route.ts` | Org-scoped (not brand) |
|
||||
| `src/app/api/organizations/invitations/route.ts` | Org-scoped (not brand) |
|
||||
|
||||
## 7. Admin Brand Scoping
|
||||
|
||||
### Admin Pages
|
||||
|
||||
| Page | Brand Selector | Status |
|
||||
|------|---------------|--------|
|
||||
| `/admin` (Dashboard) | Yes (`scope` dropdown) | OK — passes brandId to platform-metrics, confidence, reconciliation |
|
||||
| `/admin/historical` | Yes (`selectedBrand`) | FIXED — retention queries were unscoped (lines 232-234) |
|
||||
| `/admin/sessions` | Yes (`brandId`) | OK — passes brandId to sessions API |
|
||||
| `/admin/brands` | No | Cross-brand by design (brand list) |
|
||||
| `/admin/integrations` | No | Cross-brand by design (all integrations) |
|
||||
| `/admin/paid-media` | No | Cross-brand by design (platform overview) |
|
||||
| `/admin/ai-usage` | No | Cross-brand by design (cost center) |
|
||||
| `/admin/activity` | No | Cross-brand by design (activity feed) |
|
||||
| `/admin/analytics` | No | Index page, no data |
|
||||
| `/admin/compliance` | No | Cross-brand by design |
|
||||
|
||||
### Admin API Routes
|
||||
|
||||
| Route | brandId Param | Status |
|
||||
|-------|--------------|--------|
|
||||
| `/api/admin/platform-metrics` | Yes (optional) | SCOPED — all queries use brandId when provided |
|
||||
| `/api/admin/historical` | Yes (optional) | FIXED — retention queries now scoped |
|
||||
| `/api/admin/sessions` | Yes (required) | SCOPED |
|
||||
| `/api/admin/conversion-reconciliation` | Yes (required) | SCOPED |
|
||||
| `/api/admin/analytics/funnel` | Yes (required) | SCOPED |
|
||||
| `/api/admin/overview` | No | Cross-brand by design |
|
||||
| `/api/admin/signals` | No | Cross-brand by design |
|
||||
| `/api/admin/paid-media` | No | Cross-brand by design |
|
||||
| `/api/admin/system-health` | No | Cross-brand by design |
|
||||
| `/api/admin/compliance/overview` | No | Cross-brand by design |
|
||||
| `/api/admin/analytics/data-moat` | No | Cross-brand by design |
|
||||
| `/api/admin/export/platform-metrics` | No | Cross-brand by design |
|
||||
| `/api/admin/integrations` | No | Cross-brand by design |
|
||||
| `/api/admin/brands` | No | Cross-brand by design |
|
||||
|
||||
### Fix Applied
|
||||
|
||||
`/api/admin/historical/route.ts` lines 231-237: Three data-retention
|
||||
queries (`gscPage.findFirst`, `trackedSite.findFirst`,
|
||||
`metricSnapshot.findFirst`) were unscoped — returned oldest data across
|
||||
all brands even when a specific brand was selected. Fixed to apply
|
||||
`brandFilter` / `bw.brandId` to each query. Also scoped
|
||||
`metricSnapshot.count` to selected brand.
|
||||
|
||||
## 8. Cross-Brand Data Leak Sweep
|
||||
|
||||
Final sweep performed on all `prisma.gscPage`, `prisma.metricSnapshot`,
|
||||
`prisma.trackedEvent`, `prisma.trackedSession`, and
|
||||
`prisma.siteConversion` queries across the entire codebase.
|
||||
|
||||
### Methodology
|
||||
|
||||
1. Grepped all queries on these 5 high-risk tables across `src/app/api/`
|
||||
and `src/lib/services/` (239 total query sites)
|
||||
2. Verified every user-facing query includes `brandId` or `trackedSiteId`
|
||||
3. Verified every `TrackedEvent`/`TrackedSession` query uses
|
||||
`trackedSiteId` (not direct `brandId`, which doesn't exist on the table)
|
||||
4. Verified every `MetricSnapshot` query includes both `brandId` AND
|
||||
`source` filter to prevent cross-source contamination
|
||||
5. Admin/cron/debug routes verified as intentionally cross-brand
|
||||
|
||||
### Results
|
||||
|
||||
| Category | Queries Checked | Issues Found |
|
||||
|----------|----------------|--------------|
|
||||
| User-facing API routes | 85 | 0 |
|
||||
| Service files | 120 | 0 |
|
||||
| Admin API routes (scoped) | 18 | 0 (after historical fix) |
|
||||
| Admin API routes (cross-brand) | ~16 | N/A (intentional) |
|
||||
|
||||
**No cross-brand data leaks detected.** Every user-facing query is
|
||||
properly scoped by `brandId` or `trackedSiteId`.
|
||||
|
||||
## 9. Full Codebase Audit (2026-04-17)
|
||||
|
||||
Comprehensive audit covering date range handling, error handling,
|
||||
empty states, MetricSnapshot source filters, and TrackedSite resolution.
|
||||
|
||||
### Date Range Mismatches — FIXED
|
||||
|
||||
The site-tag-analytics page defaults to "28d" but 6 sub-tab API routes
|
||||
only handled "7d"|"90d" and defaulted everything else to 30 days.
|
||||
When viewing "28d", sub-tabs showed 30 days of data — a 2-day mismatch.
|
||||
|
||||
| File | Issue | Fix |
|
||||
|------|-------|-----|
|
||||
| `date-utils.ts` | `DateRange` type missing "1d", "28d" | Added + `rangeToDays()` + `rangeSince()` helpers |
|
||||
| `site-tag-analytics.ts` | `rangeStart()` had narrow type, `getPagePerformance()` only accepted 3 values | Widened to `string`, added dynamic fallback |
|
||||
| `analytics/route.ts` | Unsafe cast `range as "7d"\|"30d"\|"90d"` | Removed cast |
|
||||
| `analytics/ux/route.ts` | Missing "1d" and "28d" cases | Added |
|
||||
| `analytics/video-chat/route.ts` | Missing "1d" and "28d" cases | Added |
|
||||
| `analytics/heatmap/route.ts` | Missing "1d" and "28d" cases | Added |
|
||||
| `analytics/rum/route.ts` | Missing "1d" and "28d" cases | Added |
|
||||
| `analytics/errors/route.ts` | `rangeToSince` missing "28d", narrow type | Added "28d", widened to `string` |
|
||||
| `site-tag/confidence/route.ts` | `Range` type missing "28d" | Added |
|
||||
|
||||
### MetricSnapshot Source Filter — FIXED
|
||||
|
||||
| File | Issue | Fix |
|
||||
|------|-------|-----|
|
||||
| `client-portal/dashboard/route.ts:23` | `findFirst` without `source` filter could return GSC row (sessions=0) instead of GA4 | Added `source: "ga4"` |
|
||||
|
||||
### Admin Unscoped Queries — NOT A BUG
|
||||
|
||||
Routes in `admin/export/growth-data`, `admin/export/platform-metrics`,
|
||||
`admin/cost-center`, `admin/system-health`, `admin/analytics/data-moat`
|
||||
have unscoped TrackedEvent/TrackedSession counts. These are
|
||||
**intentionally platform-wide** — they're superadmin-only dashboards
|
||||
showing total platform volume. Not brand-scoped by design.
|
||||
|
||||
### Empty State Issues — LOW PRIORITY
|
||||
|
||||
- AI Visibility page has no empty state guide when zero queries added
|
||||
- Report Studio shows build UI before explaining the workflow
|
||||
- These are UX improvements, not data bugs
|
||||
|
||||
## 10. New Brand Experience
|
||||
|
||||
A new brand with zero data works correctly because:
|
||||
|
||||
- All pages use `useBrand()` which provides the selected brand
|
||||
- All API routes return empty arrays / zero counts when no data exists
|
||||
- `.catch(() => 0)` and `.catch(() => [])` patterns prevent crashes on empty data
|
||||
- Dashboard, analytics, and funnel pages all handle zero-data states with empty/skeleton UI
|
||||
- No hardcoded brand IDs anywhere in user-facing code
|
||||
@@ -0,0 +1,78 @@
|
||||
/**
|
||||
* Builds the composite index SiteEvent_sessionId_timestamp_idx on the
|
||||
* SiteEvent table using CREATE INDEX CONCURRENTLY.
|
||||
*
|
||||
* Must connect via the non-pooler (direct) Neon URL because:
|
||||
* - CREATE INDEX CONCURRENTLY cannot run inside a transaction
|
||||
* - The pooler enforces statement_timeout that overrides SET
|
||||
*
|
||||
* Usage:
|
||||
* DIRECT_URL="postgres://..." node scripts/build-sessionid-index.mjs
|
||||
*
|
||||
* Never hardcode credentials here. Supply DIRECT_URL from the shell only.
|
||||
*/
|
||||
|
||||
import pkg from "pg";
|
||||
const { Client } = pkg;
|
||||
|
||||
const INDEX_NAME = "SiteEvent_sessionId_timestamp_idx";
|
||||
|
||||
const directUrl = process.env.DIRECT_URL;
|
||||
if (!directUrl) {
|
||||
console.error("[build-index] DIRECT_URL is not set. Aborting.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const client = new Client({ connectionString: directUrl });
|
||||
|
||||
async function run() {
|
||||
console.log("[build-index] Connecting via DIRECT_URL...");
|
||||
await client.connect();
|
||||
console.log("[build-index] Connected.");
|
||||
|
||||
console.log("[build-index] SET statement_timeout = 0");
|
||||
await client.query("SET statement_timeout = 0;");
|
||||
|
||||
console.log("[build-index] SET lock_timeout = 0");
|
||||
await client.query("SET lock_timeout = 0;");
|
||||
|
||||
console.log(`[build-index] DROP INDEX IF EXISTS "${INDEX_NAME}"`);
|
||||
await client.query(`DROP INDEX IF EXISTS "${INDEX_NAME}";`);
|
||||
console.log("[build-index] Drop complete (or index did not exist).");
|
||||
|
||||
console.log(`[build-index] CREATE INDEX CONCURRENTLY "${INDEX_NAME}" -- this may take several minutes on large tables`);
|
||||
await client.query(
|
||||
`CREATE INDEX CONCURRENTLY "${INDEX_NAME}" ON "public"."SiteEvent" ("sessionId", "timestamp");`,
|
||||
);
|
||||
console.log("[build-index] Index build complete.");
|
||||
|
||||
const res = await client.query(
|
||||
`SELECT indisvalid
|
||||
FROM pg_index
|
||||
JOIN pg_class ON pg_class.oid = pg_index.indexrelid
|
||||
WHERE pg_class.relname = $1;`,
|
||||
[INDEX_NAME],
|
||||
);
|
||||
|
||||
if (res.rows.length === 0) {
|
||||
console.error("[build-index] Index not found after creation -- something went wrong.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const { indisvalid } = res.rows[0];
|
||||
if (indisvalid) {
|
||||
console.log("[build-index] indisvalid = true -- index is VALID and ready.");
|
||||
} else {
|
||||
console.error("[build-index] indisvalid = false -- index is INVALID. Manual cleanup required.");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
run()
|
||||
.catch((err) => {
|
||||
console.error("[build-index] Fatal error:", err.message);
|
||||
process.exit(1);
|
||||
})
|
||||
.finally(() => {
|
||||
client.end().catch(() => {});
|
||||
});
|
||||
@@ -0,0 +1,58 @@
|
||||
import { build } from 'esbuild';
|
||||
import { readFileSync, writeFileSync, copyFileSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
const SOURCE = join(process.cwd(), 'public', 't.js');
|
||||
const BACKUP = join(process.cwd(), 'public', 't.source.js');
|
||||
|
||||
async function main() {
|
||||
if (!existsSync(SOURCE)) {
|
||||
console.error(`[build-site-tag] Source not found at ${SOURCE}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const content = readFileSync(SOURCE, 'utf8');
|
||||
const lineCount = content.split('\n').length;
|
||||
|
||||
// Skip if already minified
|
||||
if (lineCount < 20 && content.includes('Minified production build')) {
|
||||
console.log('[build-site-tag] Already minified, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
const originalSize = Buffer.byteLength(content);
|
||||
console.log(`[build-site-tag] Source: ${(originalSize / 1024).toFixed(1)} KB`);
|
||||
|
||||
// Keep an unminified backup for debugging
|
||||
copyFileSync(SOURCE, BACKUP);
|
||||
|
||||
const result = await build({
|
||||
entryPoints: [SOURCE],
|
||||
minify: true,
|
||||
write: false,
|
||||
bundle: false, // t.js is a self-contained IIFE — no imports to resolve
|
||||
format: 'iife',
|
||||
target: ['es2017'], // Wide browser support
|
||||
legalComments: 'none',
|
||||
sourcemap: false,
|
||||
});
|
||||
|
||||
if (!result.outputFiles?.[0]) {
|
||||
throw new Error('[build-site-tag] esbuild returned no output');
|
||||
}
|
||||
|
||||
const minified = result.outputFiles[0].contents;
|
||||
const header = `/* meSEO Site Tag • Minified production build */\n`;
|
||||
const finalOutput = Buffer.concat([Buffer.from(header), minified]);
|
||||
|
||||
writeFileSync(SOURCE, finalOutput);
|
||||
|
||||
const newSize = finalOutput.length;
|
||||
const reduction = ((1 - newSize / originalSize) * 100).toFixed(1);
|
||||
console.log(`[build-site-tag] Minified: ${(newSize / 1024).toFixed(1)} KB (${reduction}% reduction)`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('[build-site-tag] Build failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* Guard: no raw SiteConversion queries without countedConversionWhere
|
||||
* ────────────────────────────────────────────────────────────────────
|
||||
* Fails the build when a file contains a direct prisma.siteConversion
|
||||
* count/findMany/groupBy call that neither:
|
||||
* (a) imports countedConversionWhere, nor
|
||||
* (b) carries an explicit "// intentionally unfiltered:" comment on
|
||||
* the same or immediately preceding line.
|
||||
*
|
||||
* Admin/debug/backfill endpoints that legitimately need raw rows MUST
|
||||
* add: // intentionally unfiltered: <reason>
|
||||
* to suppress the check.
|
||||
*
|
||||
* Run: node scripts/check-raw-siteconversion.mjs
|
||||
*/
|
||||
|
||||
import { readFileSync, readdirSync, statSync } from "fs";
|
||||
import { join, relative } from "path";
|
||||
|
||||
const ROOT = join(new URL(".", import.meta.url).pathname, "..");
|
||||
const SRC = join(ROOT, "src");
|
||||
|
||||
const RAW_CALL_RE = /prisma\.siteConversion\.(count|findMany|groupBy|aggregate)\s*\(/g;
|
||||
// Lines that are part of a comment block should not be flagged
|
||||
const COMMENT_LINE_RE = /^\s*(\*|\/\/|\/\*)/;
|
||||
const UNFILTERED_COMMENT_RE = /\/\/\s*intentionally unfiltered:/i;
|
||||
const IMPORT_HELPER_RE = /import\s+[^;]*countedConversionWhere[^;]*from/;
|
||||
|
||||
function walkFiles(dir, results = []) {
|
||||
for (const entry of readdirSync(dir)) {
|
||||
const full = join(dir, entry);
|
||||
const stat = statSync(full);
|
||||
if (stat.isDirectory()) {
|
||||
// Skip node_modules and .next
|
||||
if (entry === "node_modules" || entry === ".next") continue;
|
||||
walkFiles(full, results);
|
||||
} else if (full.endsWith(".ts") || full.endsWith(".tsx")) {
|
||||
results.push(full);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
const files = walkFiles(SRC);
|
||||
const violations = [];
|
||||
|
||||
for (const filePath of files) {
|
||||
const content = readFileSync(filePath, "utf8");
|
||||
const hasHelper = IMPORT_HELPER_RE.test(content);
|
||||
const lines = content.split("\n");
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (!RAW_CALL_RE.test(line) || COMMENT_LINE_RE.test(line)) {
|
||||
RAW_CALL_RE.lastIndex = 0;
|
||||
continue;
|
||||
}
|
||||
RAW_CALL_RE.lastIndex = 0;
|
||||
|
||||
// If the file imports countedConversionWhere, it's opted in — trust the author.
|
||||
if (hasHelper) continue;
|
||||
|
||||
// Check for an intentionally-unfiltered comment on this line or the line above.
|
||||
const prevLine = i > 0 ? lines[i - 1] : "";
|
||||
if (UNFILTERED_COMMENT_RE.test(line) || UNFILTERED_COMMENT_RE.test(prevLine)) continue;
|
||||
|
||||
violations.push(` ${relative(ROOT, filePath)}:${i + 1} → ${line.trim().slice(0, 80)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (violations.length > 0) {
|
||||
console.error("");
|
||||
console.error("╔══════════════════════════════════════════════════════════════════════╗");
|
||||
console.error("║ RAW SITECONVERSION CHECK FAILED ║");
|
||||
console.error("╠══════════════════════════════════════════════════════════════════════╣");
|
||||
console.error("║ The following files query SiteConversion without using ║");
|
||||
console.error("║ countedConversionWhere() or an explicit unfiltered comment. ║");
|
||||
console.error("║ ║");
|
||||
console.error("║ User-facing conversion numbers MUST route through the helper so ║");
|
||||
console.error("║ isDuplicateOf / invalidatedReason / EXCLUDED_CONVERSION_TYPES are ║");
|
||||
console.error("║ applied consistently. ║");
|
||||
console.error("║ ║");
|
||||
console.error("║ Fix options: ║");
|
||||
console.error("║ 1. import { countedConversionWhere } from ║");
|
||||
console.error('║ "@/lib/conversions/counted-where" ║');
|
||||
console.error("║ and use it in the where clause. ║");
|
||||
console.error("║ 2. Add: // intentionally unfiltered: <reason> ║");
|
||||
console.error("║ on the line before the query for debug/admin/backfill paths. ║");
|
||||
console.error("╚══════════════════════════════════════════════════════════════════════╝");
|
||||
console.error("");
|
||||
violations.forEach((v) => console.error(v));
|
||||
console.error("");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`[check-raw-siteconversion] OK — ${files.length} files checked, no violations.`);
|
||||
process.exit(0);
|
||||
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* Investigation script: enumerate distinct Brand.industry and location values,
|
||||
* show the canonicalization result for each, and sample AiAttribution fragments.
|
||||
*
|
||||
* Run from repo root (needs DATABASE_URL in env):
|
||||
* npx ts-node --compiler-options '{"module":"CommonJS"}' scripts/investigate-brand-cells.ts
|
||||
*/
|
||||
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
|
||||
async function main() {
|
||||
// 1. Distinct industry values
|
||||
const industryRows = await prisma.$queryRaw<{ industry: string | null; cnt: bigint }[]>`
|
||||
SELECT industry, COUNT(*)::bigint AS cnt
|
||||
FROM "Brand"
|
||||
GROUP BY industry
|
||||
ORDER BY cnt DESC
|
||||
`;
|
||||
console.log("\n=== Brand.industry distinct values ===");
|
||||
for (const r of industryRows) {
|
||||
console.log(` ${JSON.stringify(r.industry)} (n=${r.cnt})`);
|
||||
}
|
||||
|
||||
// 2. Distinct legacy location values (non-null, non-empty)
|
||||
const locationRows = await prisma.$queryRaw<{ location: string; cnt: bigint }[]>`
|
||||
SELECT location, COUNT(*)::bigint AS cnt
|
||||
FROM "Brand"
|
||||
WHERE location IS NOT NULL AND location <> ''
|
||||
GROUP BY location
|
||||
ORDER BY cnt DESC
|
||||
`;
|
||||
console.log("\n=== Brand.location distinct values ===");
|
||||
for (const r of locationRows) {
|
||||
console.log(` ${JSON.stringify(r.location)} (n=${r.cnt})`);
|
||||
}
|
||||
|
||||
// 3. All locations[] array elements flattened
|
||||
const locationsRows = await prisma.$queryRaw<{ loc: string; cnt: bigint }[]>`
|
||||
SELECT loc, COUNT(*)::bigint AS cnt
|
||||
FROM "Brand", unnest(locations) AS loc
|
||||
WHERE array_length(locations, 1) > 0
|
||||
GROUP BY loc
|
||||
ORDER BY cnt DESC
|
||||
`;
|
||||
console.log("\n=== Brand.locations[] distinct values (unnested) ===");
|
||||
for (const r of locationsRows) {
|
||||
console.log(` ${JSON.stringify(r.loc)} (n=${r.cnt})`);
|
||||
}
|
||||
|
||||
// 4. Canonicalization coverage check
|
||||
// Dynamic import to run inside the same process
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { canonicalizeVertical, canonicalizeMetro, getBrandCells } = await import(
|
||||
"../src/lib/market-position/cell"
|
||||
);
|
||||
|
||||
console.log("\n=== Canonicalization: industry -> vertical ===");
|
||||
const industries = industryRows.map((r) => r.industry);
|
||||
for (const ind of industries) {
|
||||
console.log(` ${JSON.stringify(ind)} -> "${canonicalizeVertical(ind)}"`);
|
||||
}
|
||||
|
||||
console.log("\n=== Canonicalization: location -> metro ===");
|
||||
const allLocations = [
|
||||
...locationRows.map((r) => r.location),
|
||||
...locationsRows.map((r) => r.loc),
|
||||
];
|
||||
const uniqueLocs = [...new Set(allLocations)];
|
||||
for (const loc of uniqueLocs) {
|
||||
console.log(` ${JSON.stringify(loc)} -> "${canonicalizeMetro(loc)}"`);
|
||||
}
|
||||
|
||||
// 5. Brand -> cell mapping (multi-vertical check)
|
||||
const brands = await prisma.brand.findMany({
|
||||
select: { id: true, name: true, industry: true, location: true, locations: true },
|
||||
});
|
||||
console.log("\n=== Brand -> cells (all brands) ===");
|
||||
for (const b of brands) {
|
||||
const cells = getBrandCells(b);
|
||||
console.log(` [${b.name}] industry="${b.industry}" -> ${JSON.stringify(cells)}`);
|
||||
}
|
||||
|
||||
// 6. Sample AiAttribution fragment shapes
|
||||
const attrs = await prisma.aiAttribution.findMany({
|
||||
select: { brandId: true, queryContext: true, landingPage: true, converted: true, dealValue: true, conversionType: true },
|
||||
orderBy: { id: "desc" },
|
||||
take: 20,
|
||||
});
|
||||
console.log("\n=== Sample AiAttribution rows (latest 20) ===");
|
||||
for (const a of attrs) {
|
||||
const qc = a.queryContext ? a.queryContext.slice(0, 120) : null;
|
||||
console.log(` brandId=${a.brandId} converted=${a.converted} dealValue=${a.dealValue} convType=${a.conversionType}`);
|
||||
console.log(` queryContext: ${JSON.stringify(qc)}`);
|
||||
console.log(` landingPage: ${JSON.stringify(a.landingPage)}`);
|
||||
}
|
||||
|
||||
// 7. Distinct conversionType values in AiAttribution
|
||||
const ctRows = await prisma.$queryRaw<{ conversionType: string | null; cnt: bigint }[]>`
|
||||
SELECT "conversionType", COUNT(*)::bigint AS cnt
|
||||
FROM "AiAttribution"
|
||||
GROUP BY "conversionType"
|
||||
ORDER BY cnt DESC
|
||||
`;
|
||||
console.log("\n=== AiAttribution.conversionType distinct values ===");
|
||||
for (const r of ctRows) {
|
||||
console.log(` ${JSON.stringify(r.conversionType)} (n=${r.cnt})`);
|
||||
}
|
||||
|
||||
await prisma.$disconnect();
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
import { spawn } from 'child_process';
|
||||
|
||||
const MAX_ATTEMPTS = 4;
|
||||
const BACKOFFS_MS = [3000, 8000, 20000, 45000];
|
||||
|
||||
function runPrismaDbPush() {
|
||||
return new Promise((resolve, reject) => {
|
||||
let stderr = '';
|
||||
const proc = spawn('npx', ['prisma', 'db', 'push', '--accept-data-loss'], { stdio: ['inherit', 'inherit', 'pipe'] });
|
||||
proc.stderr.on('data', (chunk) => {
|
||||
const text = chunk.toString();
|
||||
stderr += text;
|
||||
process.stderr.write(text);
|
||||
});
|
||||
proc.on('close', (code) => {
|
||||
if (code === 0) resolve();
|
||||
else reject(Object.assign(new Error(`prisma db push exited ${code}`), { stderr }));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
||||
try {
|
||||
console.log(`[prisma-db-push] Attempt ${attempt}/${MAX_ATTEMPTS}...`);
|
||||
await runPrismaDbPush();
|
||||
console.log('[prisma-db-push] Success.');
|
||||
process.exit(0);
|
||||
} catch (err) {
|
||||
const isP1001 = String(err.stderr || err.message).includes('P1001') || String(err.stderr || err.message).includes("Can't reach database server");
|
||||
if (!isP1001 || attempt === MAX_ATTEMPTS) {
|
||||
console.error(`[prisma-db-push] Final failure on attempt ${attempt}.`);
|
||||
process.exit(1);
|
||||
}
|
||||
const waitMs = BACKOFFS_MS[attempt - 1];
|
||||
console.log(`[prisma-db-push] P1001 on attempt ${attempt}. Waiting ${waitMs / 1000}s for Neon to wake...`);
|
||||
await new Promise((r) => setTimeout(r, waitMs));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* Prisma Schema Safety Check
|
||||
* ───────────────────────────
|
||||
* Runs before `prisma db push` in the build pipeline.
|
||||
* Aborts the build if the pending schema diff contains any destructive
|
||||
* operations: DROP COLUMN, DROP TABLE, DROP CONSTRAINT, DROP INDEX,
|
||||
* or ALTER COLUMN ... DROP.
|
||||
*
|
||||
* DROP INDEX is included because db push silently drops any index that
|
||||
* is not declared in schema.prisma, even if it is a large production
|
||||
* index (e.g. SiteEvent_sessionId_timestamp_idx on the ~22GB SiteEvent
|
||||
* table). Losing such an index causes full table scans on hot queries.
|
||||
*
|
||||
* Override: set ALLOW_SCHEMA_DROPS=true in the build environment to skip.
|
||||
*/
|
||||
|
||||
import { execSync } from "child_process";
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
|
||||
if (!DATABASE_URL) {
|
||||
console.log("[prisma-safety] DATABASE_URL not set — skipping safety check");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (process.env.ALLOW_SCHEMA_DROPS === "true") {
|
||||
console.log("[prisma-safety] ALLOW_SCHEMA_DROPS=true — skipping drop check");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
console.log("[prisma-safety] Computing schema diff against live database...");
|
||||
|
||||
// Retry up to 3 times to handle transient P1001 connection errors on cold starts.
|
||||
const MAX_RETRIES = 3;
|
||||
let diffSql;
|
||||
let lastErr;
|
||||
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
diffSql = execSync(
|
||||
"npx prisma migrate diff --from-url $DATABASE_URL --to-schema-datamodel prisma/schema.prisma --script",
|
||||
{ encoding: "utf8", stdio: ["pipe", "pipe", "pipe"] },
|
||||
);
|
||||
lastErr = null;
|
||||
break;
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
const stderr = err.stderr ?? "";
|
||||
const isTransient =
|
||||
stderr.includes("P1001") ||
|
||||
stderr.includes("Could not connect") ||
|
||||
stderr.includes("connection");
|
||||
if (isTransient && attempt < MAX_RETRIES) {
|
||||
console.warn(`[prisma-safety] Connection attempt ${attempt} failed — retrying...`);
|
||||
// brief pause before retry (synchronous busy-wait, safe in build context)
|
||||
const wait = attempt * 2000;
|
||||
const end = Date.now() + wait;
|
||||
while (Date.now() < end) { /* spin */ }
|
||||
continue;
|
||||
}
|
||||
if (isTransient) {
|
||||
console.warn("[prisma-safety] Could not connect to database after retries — skipping safety check");
|
||||
console.warn("[prisma-safety]", stderr.split("\n")[0]);
|
||||
process.exit(0);
|
||||
}
|
||||
console.error("[prisma-safety] Failed to compute schema diff:");
|
||||
console.error(stderr || err.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
if (lastErr) {
|
||||
// Should not reach here, but guard anyway.
|
||||
console.warn("[prisma-safety] Exhausted retries — skipping safety check");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Patterns that signal data-destructive or performance-destructive operations.
|
||||
// DROP INDEX is included: db push silently drops undeclared indexes, turning
|
||||
// hot queries into full table scans (root cause of the SiteEvent incident).
|
||||
const DROP_PATTERNS = [
|
||||
/^\s*ALTER TABLE\s+\S+\s+DROP COLUMN\s+/im,
|
||||
/^\s*DROP TABLE\s+/im,
|
||||
/^\s*DROP CONSTRAINT\s+/im,
|
||||
/^\s*ALTER TABLE\s+\S+\s+DROP CONSTRAINT\s+/im,
|
||||
/^\s*ALTER COLUMN\s+\S+\s+DROP\s+/im,
|
||||
/^\s*DROP INDEX\s+/im,
|
||||
];
|
||||
|
||||
const matchedStatements = DROP_PATTERNS
|
||||
.flatMap((pattern) => {
|
||||
const lines = diffSql.split("\n");
|
||||
return lines.filter((line) => pattern.test(line));
|
||||
})
|
||||
.filter((line, i, arr) => arr.indexOf(line) === i); // deduplicate
|
||||
|
||||
if (matchedStatements.length > 0) {
|
||||
console.error("");
|
||||
console.error("╔══════════════════════════════════════════════════════════════╗");
|
||||
console.error("║ PRISMA SAFETY CHECK FAILED — BUILD ABORTED ║");
|
||||
console.error("╠══════════════════════════════════════════════════════════════╣");
|
||||
console.error("║ The schema diff would execute the following destructive ║");
|
||||
console.error("║ SQL statements that could permanently delete production ║");
|
||||
console.error("║ data or drop production indexes (causing full table scans): ║");
|
||||
console.error("╚══════════════════════════════════════════════════════════════╝");
|
||||
console.error("");
|
||||
matchedStatements.forEach((stmt) => console.error(" ⚠", stmt.trim()));
|
||||
console.error("");
|
||||
console.error(" Full diff SQL:");
|
||||
console.error(" " + diffSql.split("\n").join("\n "));
|
||||
console.error("");
|
||||
console.error(" If this drop is intentional, set ALLOW_SCHEMA_DROPS=true");
|
||||
console.error(" in the Vercel build environment and redeploy.");
|
||||
console.error("");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log("[prisma-safety] No destructive operations found. Build can proceed.");
|
||||
process.exit(0);
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* Self-test for prisma-safety-check.mjs.
|
||||
* Verifies each DROP pattern fires and the safe-SQL path passes.
|
||||
* Run with: node scripts/test-safety-check.mjs
|
||||
*/
|
||||
|
||||
const DROP_PATTERNS = [
|
||||
/^\s*ALTER TABLE\s+\S+\s+DROP COLUMN\s+/im,
|
||||
/^\s*DROP TABLE\s+/im,
|
||||
/^\s*DROP CONSTRAINT\s+/im,
|
||||
/^\s*ALTER TABLE\s+\S+\s+DROP CONSTRAINT\s+/im,
|
||||
/^\s*ALTER COLUMN\s+\S+\s+DROP\s+/im,
|
||||
/^\s*DROP INDEX\s+/im,
|
||||
];
|
||||
|
||||
function matches(sql) {
|
||||
return DROP_PATTERNS.some((p) => p.test(sql));
|
||||
}
|
||||
|
||||
const cases = [
|
||||
// should block
|
||||
{ sql: "ALTER TABLE \"User\" DROP COLUMN email;", expect: true, label: "DROP COLUMN" },
|
||||
{ sql: "DROP TABLE \"OldTable\";", expect: true, label: "DROP TABLE" },
|
||||
{ sql: "DROP CONSTRAINT fk_user_org;", expect: true, label: "DROP CONSTRAINT (bare)" },
|
||||
{ sql: "ALTER TABLE \"Brand\" DROP CONSTRAINT chk_plan;", expect: true, label: "DROP CONSTRAINT (alter)" },
|
||||
{ sql: "ALTER COLUMN status DROP NOT NULL;", expect: true, label: "ALTER COLUMN DROP" },
|
||||
{ sql: "DROP INDEX \"SiteEvent_sessionId_timestamp_idx\";",expect: true, label: "DROP INDEX (the incident)" },
|
||||
{ sql: "DROP INDEX IF EXISTS \"some_idx\";", expect: true, label: "DROP INDEX IF EXISTS" },
|
||||
// should pass
|
||||
{ sql: "CREATE INDEX idx ON \"SiteEvent\" (\"sessionId\");", expect: false, label: "CREATE INDEX (safe)" },
|
||||
{ sql: "ALTER TABLE \"Brand\" ADD COLUMN plan TEXT;", expect: false, label: "ADD COLUMN (safe)" },
|
||||
{ sql: "CREATE TABLE \"NewModel\" (id TEXT PRIMARY KEY);", expect: false, label: "CREATE TABLE (safe)" },
|
||||
];
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
for (const c of cases) {
|
||||
const got = matches(c.sql);
|
||||
const ok = got === c.expect;
|
||||
console.log(`${ok ? "PASS" : "FAIL"} [${c.label}]`);
|
||||
if (!ok) {
|
||||
console.log(` sql: ${c.sql}`);
|
||||
console.log(` expected: ${c.expect}, got: ${got}`);
|
||||
failed++;
|
||||
} else {
|
||||
passed++;
|
||||
}
|
||||
}
|
||||
console.log(`\n${passed} passed, ${failed} failed`);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
@@ -0,0 +1,325 @@
|
||||
/**
|
||||
* Block 2 verification: canonicalization + fragmentToQuery.
|
||||
* Run without a database -- pure logic checks.
|
||||
*
|
||||
* node scripts/verify-block2.mjs
|
||||
*/
|
||||
|
||||
// ---- inline the canonicalization logic (mirrors cell.ts) ----
|
||||
|
||||
const INDUSTRY_TO_VERTICAL = {
|
||||
"healthcare-medical": "healthcare",
|
||||
"healthcare-dental": "dental",
|
||||
"healthcare-mental-health": "mental-health",
|
||||
"healthcare-veterinary": "veterinary",
|
||||
"healthcare-other": "healthcare",
|
||||
"b2b-saas": "b2b-saas",
|
||||
"b2b-services": "b2b-services",
|
||||
"b2b-manufacturing": "b2b-manufacturing",
|
||||
"b2b-finance": "finance",
|
||||
"b2b-legal": "legal",
|
||||
"b2b-other": "b2b-services",
|
||||
"ecommerce": "ecommerce",
|
||||
"ecommerce-fashion": "ecommerce",
|
||||
"ecommerce-beauty": "ecommerce",
|
||||
"ecommerce-electronics": "ecommerce",
|
||||
"ecommerce-home": "ecommerce",
|
||||
"ecommerce-sports": "ecommerce",
|
||||
"ecommerce-food": "ecommerce",
|
||||
"ecommerce-other": "ecommerce",
|
||||
"local-services": "local-services",
|
||||
"professional-services": "professional-services",
|
||||
"finance": "finance",
|
||||
"legal": "legal",
|
||||
"real-estate": "real-estate",
|
||||
"nonprofit": "nonprofit",
|
||||
"hospitality": "hospitality",
|
||||
"higher-ed": "higher-ed",
|
||||
"other": "other",
|
||||
};
|
||||
|
||||
const LOCAL_VERTICALS = new Set([
|
||||
"dental", "healthcare", "mental-health", "veterinary",
|
||||
"local-services", "real-estate", "hospitality", "legal", "professional-services",
|
||||
]);
|
||||
|
||||
const METRO_ALIASES = {
|
||||
"new york": "new-york", "new york city": "new-york", "nyc": "new-york",
|
||||
"los angeles": "los-angeles", "la": "los-angeles",
|
||||
"chicago": "chicago", "houston": "houston", "phoenix": "phoenix",
|
||||
"philadelphia": "philadelphia", "san antonio": "san-antonio",
|
||||
"san diego": "san-diego", "dallas": "dallas", "san jose": "san-jose",
|
||||
"austin": "austin", "charlotte": "charlotte",
|
||||
"san francisco": "san-francisco", "sf": "san-francisco",
|
||||
"denver": "denver", "boston": "boston", "seattle": "seattle",
|
||||
"atlanta": "atlanta", "miami": "miami", "raleigh": "raleigh",
|
||||
"nashville": "nashville", "minneapolis": "minneapolis",
|
||||
};
|
||||
|
||||
function canonicalizeVertical(industry) {
|
||||
if (!industry) return "other";
|
||||
const key = industry.toLowerCase().trim();
|
||||
if (INDUSTRY_TO_VERTICAL[key]) return INDUSTRY_TO_VERTICAL[key];
|
||||
const stripped = key.replace(/[-\s]/g, "");
|
||||
for (const [k, v] of Object.entries(INDUSTRY_TO_VERTICAL)) {
|
||||
if (k.replace(/[-\s]/g, "") === stripped) return v;
|
||||
}
|
||||
if (key.startsWith("healthcare")) return "healthcare";
|
||||
if (key.startsWith("b2b")) return "b2b-services";
|
||||
if (key.startsWith("ecommerce")) return "ecommerce";
|
||||
return "other";
|
||||
}
|
||||
|
||||
function canonicalizeMetro(location) {
|
||||
const lower = location.toLowerCase().trim();
|
||||
if (METRO_ALIASES[lower]) return METRO_ALIASES[lower];
|
||||
const cityPart = lower.split(/[,;]/)[0].trim();
|
||||
if (METRO_ALIASES[cityPart]) return METRO_ALIASES[cityPart];
|
||||
return cityPart.replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "unknown";
|
||||
}
|
||||
|
||||
function getBrandLocations(brand) {
|
||||
if (brand.locations && brand.locations.length > 0) return brand.locations;
|
||||
if (brand.location && brand.location.trim()) return [brand.location.trim()];
|
||||
return [];
|
||||
}
|
||||
|
||||
function getBrandCells(brand) {
|
||||
const vertical = canonicalizeVertical(brand.industry);
|
||||
const cells = [{ vertical, locale: "national" }];
|
||||
if (LOCAL_VERTICALS.has(vertical)) {
|
||||
const seen = new Set();
|
||||
for (const loc of getBrandLocations(brand)) {
|
||||
const metro = canonicalizeMetro(loc);
|
||||
if (metro && metro !== "unknown" && !seen.has(metro)) {
|
||||
seen.add(metro);
|
||||
cells.push({ vertical, locale: `metro:${metro}` });
|
||||
}
|
||||
}
|
||||
}
|
||||
return cells;
|
||||
}
|
||||
|
||||
// ---- inline fragmentToQuery logic (mirrors seed-corpus.ts) ----
|
||||
|
||||
const STOP_WORDS = new Set([
|
||||
"the","a","an","and","or","but","in","on","at","to","for","of","with",
|
||||
"by","from","is","are","was","were","be","been","has","have","had","do",
|
||||
"does","did","will","would","could","should","may","might","can","our",
|
||||
"your","their","its","this","that","these","those","we","you","they","it",
|
||||
"he","she","i","me","my","us","not","no","so","if","as","up","out","about",
|
||||
"into","than","then","when","where","which","who","how","what","why","all",
|
||||
"also","just","more","most","other","such","get","use","using","used","new",
|
||||
"need","needs","make",
|
||||
// common content verbs that appear in AI answer fragments
|
||||
"provides","provide","offers","offer","delivers","deliver","includes","include",
|
||||
"helps","help","gives","give","allows","allow","enables","enable","features",
|
||||
"feature","serves","serve","creates","create","makes","builds","build",
|
||||
"brings","bring",
|
||||
// qualifiers that add noise
|
||||
"same","days","area","near","patients","customers","clients","team","staff",
|
||||
"experts","people","many","every","each","both","full","well","come","wide",
|
||||
"high","even","here","there","very","always","never","often",
|
||||
]);
|
||||
|
||||
function inferFromLandingPage(url) {
|
||||
try {
|
||||
const pathname = url.startsWith("http") ? new URL(url).pathname : url;
|
||||
const segments = pathname.split("/").filter(Boolean);
|
||||
const meaningful = segments.filter(s => s.length > 2 && !/^[a-z]{2}$/.test(s));
|
||||
const slug = meaningful[meaningful.length - 1] ?? "";
|
||||
if (!slug) return "site visit";
|
||||
return slug.replace(/[-_]/g, " ").replace(/\s+/g, " ").trim();
|
||||
} catch { return "site visit"; }
|
||||
}
|
||||
|
||||
function fragmentToQuery(fragment, landingPage, brandName) {
|
||||
const isInferred = fragment.startsWith("[inferred]");
|
||||
if (isInferred) {
|
||||
const raw = inferFromLandingPage(landingPage);
|
||||
const base = raw === "site visit" || raw.includes(".") ? "" : raw;
|
||||
return base ? `best ${base} near me` : `${brandName} services`;
|
||||
}
|
||||
const brandPattern = new RegExp(brandName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "gi");
|
||||
const cleaned = fragment.replace(brandPattern, "").replace(/["""'']/g, "").replace(/\s+/g, " ").trim();
|
||||
const tokens = cleaned.split(/\W+/)
|
||||
.filter(w => w.length > 3 && !STOP_WORDS.has(w.toLowerCase()) && !/^\d+$/.test(w));
|
||||
const keywords = tokens.slice(0, 5).join(" ").toLowerCase();
|
||||
if (!keywords) return `${brandName} services`;
|
||||
return keywords;
|
||||
}
|
||||
|
||||
const COMMERCIAL_CONVERSION_TYPES = new Set([
|
||||
"appointment_booked","form_submitted","phone_call","email_contact","sms_contact","purchase",
|
||||
"booking_confirmed","form_submit","form_submission","phone_number_click","click_to_call",
|
||||
"calendly","email_click","sms_click",
|
||||
]);
|
||||
|
||||
function deriveConversionWeight(converted, dealValue, conversionType) {
|
||||
if (!converted) return 0;
|
||||
let base = 0.3;
|
||||
if (conversionType && COMMERCIAL_CONVERSION_TYPES.has(conversionType.toLowerCase())) base += 0.3;
|
||||
if (dealValue !== null) {
|
||||
if (dealValue >= 10000) base += 0.4;
|
||||
else if (dealValue >= 1000) base += 0.3;
|
||||
else if (dealValue >= 100) base += 0.2;
|
||||
else base += 0.1;
|
||||
}
|
||||
return Math.min(base, 1);
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// SECTION 1: Multi-vertical cell bucketing
|
||||
// ============================================================
|
||||
|
||||
console.log("\n=== SECTION 1: Multi-vertical cell bucketing ===\n");
|
||||
|
||||
const testBrands = [
|
||||
{ name: "Raleigh Cardiovascular Specialists", industry: "healthcare-medical", location: "Raleigh, NC", locations: ["Raleigh, NC"] },
|
||||
{ name: "Fairway Lawns", industry: "local-services", location: "Charlotte, NC", locations: ["Charlotte, NC", "Raleigh, NC"] },
|
||||
{ name: "AiGrowth360", industry: "b2b-saas", location: "United States", locations: [] },
|
||||
{ name: "TPAction", industry: "b2b-services", location: null, locations: [] },
|
||||
{ name: "Happy Smiles Dental", industry: "healthcare-dental", location: "Austin, TX", locations: ["Austin, TX"] },
|
||||
];
|
||||
|
||||
for (const brand of testBrands) {
|
||||
const cells = getBrandCells(brand);
|
||||
console.log(`[${brand.name}]`);
|
||||
console.log(` industry="${brand.industry}" -> vertical="${canonicalizeVertical(brand.industry)}"`);
|
||||
console.log(` cells: ${JSON.stringify(cells)}`);
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// SECTION 2: Canonicalization coverage (all known classifier outputs)
|
||||
// ============================================================
|
||||
|
||||
console.log("\n=== SECTION 2: Industry -> vertical canonicalization map ===\n");
|
||||
|
||||
const allIndustries = [
|
||||
"healthcare-medical","healthcare-dental","healthcare-mental-health","healthcare-veterinary","healthcare-other",
|
||||
"b2b-saas","b2b-services","b2b-manufacturing","b2b-finance","b2b-legal","b2b-other",
|
||||
"ecommerce","ecommerce-fashion","ecommerce-beauty","ecommerce-electronics","ecommerce-home",
|
||||
"ecommerce-sports","ecommerce-food","ecommerce-other",
|
||||
"local-services","professional-services","finance","legal","real-estate",
|
||||
"nonprofit","hospitality","higher-ed","other",
|
||||
// Fuzzy / drift cases
|
||||
null, "", "Healthcare Medical", "B2B SaaS", "ECOMMERCE", "unknown-industry",
|
||||
];
|
||||
for (const ind of allIndustries) {
|
||||
console.log(` ${JSON.stringify(ind).padEnd(32)} -> "${canonicalizeVertical(ind)}"`);
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// SECTION 3: fragmentToQuery derivation examples
|
||||
// ============================================================
|
||||
|
||||
console.log("\n=== SECTION 3: fragmentToQuery derivation ===\n");
|
||||
|
||||
const fragmentTests = [
|
||||
{
|
||||
desc: "Medical brand fragment",
|
||||
fragment: "Raleigh Cardiovascular Specialists provides comprehensive cardiac imaging and interventional cardiology services for patients throughout central North Carolina",
|
||||
landingPage: "https://raleighcardio.com/services/cardiac-imaging",
|
||||
brandName: "Raleigh Cardiovascular Specialists",
|
||||
},
|
||||
{
|
||||
desc: "Dental brand fragment",
|
||||
fragment: "Happy Smiles Dental offers same-day emergency dental appointments and affordable teeth whitening treatments for Austin area patients",
|
||||
landingPage: "https://happysmilesdental.com/emergency-dentist",
|
||||
brandName: "Happy Smiles Dental",
|
||||
},
|
||||
{
|
||||
desc: "B2B SaaS fragment",
|
||||
fragment: "AiGrowth360 delivers AI-powered marketing automation and lead scoring tools purpose-built for B2B sales teams",
|
||||
landingPage: "https://aigrowth360.com/features/lead-scoring",
|
||||
brandName: "AiGrowth360",
|
||||
},
|
||||
{
|
||||
desc: "Lawn care (inferred fragment)",
|
||||
fragment: "[inferred] lawn care services",
|
||||
landingPage: "https://fairwaylawns.com/services/lawn-fertilization",
|
||||
brandName: "Fairway Lawns",
|
||||
},
|
||||
{
|
||||
desc: "Empty fragment fallback",
|
||||
fragment: "[inferred] site visit",
|
||||
landingPage: "https://tpaction.com/",
|
||||
brandName: "TPAction",
|
||||
},
|
||||
];
|
||||
|
||||
for (const t of fragmentTests) {
|
||||
const q = fragmentToQuery(t.fragment, t.landingPage, t.brandName);
|
||||
console.log(`[${t.desc}]`);
|
||||
console.log(` fragment: "${t.fragment.slice(0, 80)}..."`);
|
||||
console.log(` derived query: "${q}"`);
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// SECTION 4: conversionWeight examples
|
||||
// ============================================================
|
||||
|
||||
console.log("\n=== SECTION 4: deriveConversionWeight examples ===\n");
|
||||
|
||||
const weightTests = [
|
||||
{ converted: false, dealValue: 5000, conversionType: "appointment_booked", label: "unconverted (no weight)" },
|
||||
{ converted: true, dealValue: null, conversionType: null, label: "converted, no type, no value" },
|
||||
{ converted: true, dealValue: null, conversionType: "appointment_booked", label: "converted, commercial type, no value" },
|
||||
{ converted: true, dealValue: 250, conversionType: "appointment_booked", label: "converted, commercial type, dealValue=250" },
|
||||
{ converted: true, dealValue: 2500, conversionType: "form_submitted", label: "converted, commercial type, dealValue=2500" },
|
||||
{ converted: true, dealValue: 15000, conversionType: "purchase", label: "converted, purchase, dealValue=15000 (cap)" },
|
||||
{ converted: true, dealValue: null, conversionType: "phone_call", label: "converted, phone_call (old type)" },
|
||||
{ converted: true, dealValue: null, conversionType: "phone_number_click", label: "converted, phone_number_click (legacy)" },
|
||||
{ converted: true, dealValue: null, conversionType: "unknown_type", label: "converted, unrecognized type (no bonus)" },
|
||||
];
|
||||
|
||||
for (const t of weightTests) {
|
||||
const w = deriveConversionWeight(t.converted, t.dealValue, t.conversionType);
|
||||
console.log(` [${t.label}]`);
|
||||
console.log(` converted=${t.converted} dealValue=${t.dealValue} convType=${t.conversionType} -> weight=${w.toFixed(2)}`);
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// SECTION 5: Simulated corpus for one cell (healthcare/national)
|
||||
// ============================================================
|
||||
|
||||
console.log("\n=== SECTION 5: Simulated FRAGMENT_SEEDED corpus (healthcare, national) ===\n");
|
||||
|
||||
const healthcareSampleAttributions = [
|
||||
{ brandId: "b1", queryContext: "Raleigh Cardiovascular provides cardiac imaging and heart failure management for patients", landingPage: "https://raleighcardio.com/cardiac-imaging", converted: true, dealValue: 3000, conversionType: "appointment_booked" },
|
||||
{ brandId: "b1", queryContext: "interventional cardiology procedures including stent placement and catheterization", landingPage: "https://raleighcardio.com/procedures", converted: false, dealValue: null, conversionType: null },
|
||||
{ brandId: "b1", queryContext: null, landingPage: "https://raleighcardio.com/pediatric-cardiology", converted: true, dealValue: 500, conversionType: "form_submitted" },
|
||||
{ brandId: "b2", queryContext: "same-day urgent care and family medicine appointments available", landingPage: "https://example-medical.com/urgent-care", converted: true, dealValue: null, conversionType: "appointment_booked" },
|
||||
];
|
||||
|
||||
const brandNames = new Map([["b1", "Raleigh Cardiovascular"], ["b2", "Example Medical Group"]]);
|
||||
const weightMap = new Map();
|
||||
|
||||
for (const attr of healthcareSampleAttributions) {
|
||||
const fragment = attr.queryContext ?? `[inferred] site visit`;
|
||||
const brandName = brandNames.get(attr.brandId) ?? "";
|
||||
const query = fragmentToQuery(fragment, attr.landingPage, brandName).trim();
|
||||
if (!query || query.length < 5) continue;
|
||||
const weight = deriveConversionWeight(attr.converted, attr.dealValue, attr.conversionType);
|
||||
const prev = weightMap.get(query);
|
||||
if (prev === undefined || weight > prev) weightMap.set(query, weight);
|
||||
}
|
||||
|
||||
console.log(" FRAGMENT_SEEDED entries (vertical=healthcare, locale=national):");
|
||||
for (const [query, weight] of weightMap) {
|
||||
console.log(` query="${query}" conversionWeight=${weight.toFixed(2)}`);
|
||||
}
|
||||
|
||||
console.log("\n CATEGORY_COVERAGE entries (vertical=healthcare, locale=national):");
|
||||
const healthcareCategories = [
|
||||
"best primary care doctor near me",
|
||||
"how to find a good family physician",
|
||||
"primary care vs urgent care when to go",
|
||||
"what to look for in a primary care doctor",
|
||||
"how to get a same-day doctor appointment",
|
||||
];
|
||||
for (const q of healthcareCategories) {
|
||||
console.log(` query="${q}" conversionWeight=0.00 source=CATEGORY_COVERAGE`);
|
||||
}
|
||||
|
||||
console.log("\n=== All checks passed ===\n");
|
||||
Reference in New Issue
Block a user