diff --git a/Server/cmd/seed/profile_alpha.go b/Server/cmd/seed/profile_alpha.go index 6ceb1475..d2016a68 100644 --- a/Server/cmd/seed/profile_alpha.go +++ b/Server/cmd/seed/profile_alpha.go @@ -38,6 +38,7 @@ import ( "log" "math/rand" "os" + "sort" "strings" "time" @@ -177,6 +178,7 @@ func seedAlpha(database *db.DB) error { rng := rand.New(rand.NewSource(alphaSeed)) windowStart := alphaWindowEnd.AddDate(0, 0, -alphaWindowDays) + joined := alphaJoinTimes(rng, windowStart) tx, err := database.BeginTx(context.Background(), &sql.TxOptions{}) if err != nil { @@ -184,21 +186,21 @@ func seedAlpha(database *db.DB) error { } defer func() { _ = tx.Rollback() }() - if err := alphaInsertUsers(tx, rng, windowStart); err != nil { + if err := alphaInsertUsers(tx, rng, joined); err != nil { return err } if err := alphaInsertChannels(tx, windowStart); err != nil { return err } - pairs, err := alphaInsertDMs(tx, rng, windowStart) + pairs, err := alphaInsertDMs(tx, rng, joined) if err != nil { return err } - messageTimes, err := alphaInsertMessages(tx, rng, windowStart, pairs) + messageTimes, messageAuthors, err := alphaInsertMessages(tx, rng, windowStart, joined, pairs) if err != nil { return err } - if err := alphaInsertAttachments(tx, rng, messageTimes); err != nil { + if err := alphaInsertAttachments(tx, rng, messageTimes, messageAuthors); err != nil { return err } if err := alphaInsertReactions(tx, rng); err != nil { @@ -261,7 +263,28 @@ func writeSnapshot(database *db.DB, scrubPath, outPath string) error { // ─── Users ────────────────────────────────────────────────────────────────── -func alphaInsertUsers(tx *sql.Tx, rng *rand.Rand, windowStart time.Time) error { +// alphaJoinTimes computes every user's created_at once — the message and DM +// generators consult it so nothing is authored before its author joined (a +// fixture with time-travelling rows would poison the retention and +// account-age rehearsals that consume it; Codex on #1469, P2). Monotonic in +// the user id — the first forty accounts predate the window (an established +// server; all staff are among them), the remaining sixty join during it in +// id order — so "who exists at time t" is always the prefix 1..N. +func alphaJoinTimes(rng *rand.Rand, windowStart time.Time) []time.Time { + joined := make([]time.Time, alphaUsers+1) + for i := 1; i <= alphaUsers; i++ { + if i <= 40 { + joined[i] = windowStart.AddDate(0, 0, -60).Add(time.Duration(i) * 26 * time.Hour) + continue + } + // Spacing is 12h, jitter under 1h: monotonicity holds by construction. + into := time.Duration(i-41) * (time.Duration(alphaWindowDays) * 24 * time.Hour) / 60 + joined[i] = windowStart.Add(into).Add(time.Duration(rng.Intn(3599)) * time.Second) + } + return joined +} + +func alphaInsertUsers(tx *sql.Tx, rng *rand.Rand, joined []time.Time) error { roleOf := func(i int) int { // i is 1-based user number switch { case i <= alphaOwners: @@ -274,15 +297,6 @@ func alphaInsertUsers(tx *sql.Tx, rng *rand.Rand, windowStart time.Time) error { return 4 } } - // The first forty accounts predate the window (an established server); - // the remaining sixty joined during it, spread evenly with jitter. - joinedAt := func(i int) time.Time { - if i <= 40 { - return windowStart.AddDate(0, 0, -60).Add(time.Duration(i) * 26 * time.Hour) - } - into := time.Duration(i-41) * (time.Duration(alphaWindowDays) * 24 * time.Hour) / 60 - return windowStart.Add(into).Add(time.Duration(rng.Intn(3600)) * time.Second) - } const cols = 7 var b strings.Builder @@ -292,8 +306,13 @@ func alphaInsertUsers(tx *sql.Tx, rng *rand.Rand, windowStart time.Time) error { b.WriteString(",") } b.WriteString("(?,?,?,?,?,?,?)") - created := joinedAt(i) + created := joined[i] lastSeen := alphaWindowEnd.Add(-time.Duration(rng.Intn(72*3600)) * time.Second) + if lastSeen.Before(created) { + // A user who joined in the window's final days was last seen at + // the join, not before it. + lastSeen = created + } args = append(args, i, fmt.Sprintf("user%03d", i), alphaPasswordHash, roleOf(i), "offline", ts(created), ts(lastSeen), @@ -370,16 +389,27 @@ func alphaInsertChannels(tx *sql.Tx, windowStart time.Time) error { type dmPair struct { channelID int64 a, b int64 + // ready is when both members exist — no message in the pair may precede + // it, and it is the channel's created_at / opened_at, matching + // GetOrCreateDMChannel's create-on-first-contact shape. + ready time.Time } // alphaInsertDMs creates the 40 DM channels exactly as GetOrCreateDMChannel -// would (type 'dm', empty name, both participants, both open). -func alphaInsertDMs(tx *sql.Tx, rng *rand.Rand, windowStart time.Time) ([]dmPair, error) { +// would (type 'dm', empty name, both participants, both open). The first +// twelve pairs are drawn from the pre-window accounts so DM traffic exists +// from the window's first hour; the rest may involve joiners and only carry +// messages once both members exist. +func alphaInsertDMs(tx *sql.Tx, rng *rand.Rand, joined []time.Time) ([]dmPair, error) { seen := map[[2]int64]bool{} pairs := make([]dmPair, 0, alphaDMPairs) for len(pairs) < alphaDMPairs { - a := int64(rng.Intn(alphaUsers) + 1) - b := int64(rng.Intn(alphaUsers) + 1) + pool := alphaUsers + if len(pairs) < 12 { + pool = 40 // both members pre-window + } + a := int64(rng.Intn(pool) + 1) + b := int64(rng.Intn(pool) + 1) if a == b { continue } @@ -391,10 +421,13 @@ func alphaInsertDMs(tx *sql.Tx, rng *rand.Rand, windowStart time.Time) ([]dmPair } seen[[2]int64{a, b}] = true id := int64(alphaChannels + len(pairs) + 1) // 13..52 - opened := windowStart.Add(time.Duration(len(pairs)) * 17 * time.Hour) + ready := joined[a] + if joined[b].After(ready) { + ready = joined[b] + } if _, err := tx.Exec( `INSERT INTO channels (id, name, type, is_group, created_at) VALUES (?, '', 'dm', 0, ?)`, - id, ts(opened), + id, ts(ready), ); err != nil { return nil, fmt.Errorf("alpha: dm channel %d: %w", id, err) } @@ -405,12 +438,12 @@ func alphaInsertDMs(tx *sql.Tx, rng *rand.Rand, windowStart time.Time) ([]dmPair return nil, fmt.Errorf("alpha: dm participant: %w", err) } if _, err := tx.Exec( - `INSERT INTO dm_open_state (user_id, channel_id, opened_at) VALUES (?, ?, ?)`, u, id, ts(opened), + `INSERT INTO dm_open_state (user_id, channel_id, opened_at) VALUES (?, ?, ?)`, u, id, ts(ready), ); err != nil { return nil, fmt.Errorf("alpha: dm open state: %w", err) } } - pairs = append(pairs, dmPair{channelID: id, a: a, b: b}) + pairs = append(pairs, dmPair{channelID: id, a: a, b: b, ready: ready}) } return pairs, nil } @@ -418,9 +451,14 @@ func alphaInsertDMs(tx *sql.Tx, rng *rand.Rand, windowStart time.Time) ([]dmPair // ─── Messages ─────────────────────────────────────────────────────────────── // alphaInsertMessages writes all 20,000 messages in chronological order — -// ascending ids follow ascending time, as they would on a live server — and -// returns each message's timestamp (indexed by id-1) for the attachment rows. -func alphaInsertMessages(tx *sql.Tx, rng *rand.Rand, windowStart time.Time, pairs []dmPair) ([]time.Time, error) { +// per-bucket second offsets are sorted before ids are assigned, so ascending +// ids follow ascending timestamps exactly as insertion order does on a live +// server (Codex on #1469, P2) — and returns each message's timestamp and +// author (indexed by id-1) for the attachment rows. Authors are drawn only +// from users who exist at the message's time: the eligible set is always the +// prefix 1..maxUser because alphaJoinTimes is monotonic, and a DM pair +// carries traffic only from its ready time. +func alphaInsertMessages(tx *sql.Tx, rng *rand.Rand, windowStart time.Time, joined []time.Time, pairs []dmPair) ([]time.Time, []int64, error) { // Exactly 15% of message ids are DM messages. dmCount := int(float64(alphaMessages) * alphaDMShare) isDM := make([]bool, alphaMessages) @@ -455,8 +493,9 @@ func alphaInsertMessages(tx *sql.Tx, rng *rand.Rand, windowStart time.Time, pair return 3 } // Staff channel posters are staff (+ the override-listed user009); - // announcement channels are staff-posted; everywhere else, anyone. - pickAuthor := func(ch int64) int64 { + // announcement channels are staff-posted; everywhere else, anyone who has + // joined by the message's time (staff and user009 are pre-window). + pickAuthor := func(ch int64, maxUser int) int64 { switch ch { case 1, 2: return int64(rng.Intn(alphaOwners+alphaAdmins+alphaModerators) + 1) @@ -464,11 +503,21 @@ func alphaInsertMessages(tx *sql.Tx, rng *rand.Rand, windowStart time.Time, pair staff := []int64{1, 2, 3, 4, 5, 6, 7, 8, 9} return staff[rng.Intn(len(staff))] default: - return int64(rng.Intn(alphaUsers) + 1) + return int64(rng.Intn(maxUser) + 1) } } + // Pairs sorted by readiness, so the eligible set at any time is a prefix; + // skewedIndex over that prefix keeps the earliest (all-pre-window) pairs + // the chattiest, which is also the realistic shape. + byReady := make([]dmPair, len(pairs)) + copy(byReady, pairs) + sort.Slice(byReady, func(i, j int) bool { return byReady[i].ready.Before(byReady[j].ready) }) + times := make([]time.Time, 0, alphaMessages) + authors := make([]int64, 0, alphaMessages) + maxUser := 40 + readyPairs := 0 const cols = 5 const chunk = 500 var b strings.Builder @@ -490,13 +539,23 @@ func alphaInsertMessages(tx *sql.Tx, rng *rand.Rand, windowStart time.Time, pair for day := 0; day < alphaWindowDays; day++ { perHour := largestRemainder(weights, perDay[day]) for hour := 0; hour < 24; hour++ { - for k := 0; k < perHour[hour]; k++ { - at := windowStart.AddDate(0, 0, day). - Add(time.Duration(hour) * time.Hour). - Add(time.Duration(rng.Intn(3600)) * time.Second) + offsets := make([]int, perHour[hour]) + for k := range offsets { + offsets[k] = rng.Intn(3600) + } + sort.Ints(offsets) + base := windowStart.AddDate(0, 0, day).Add(time.Duration(hour) * time.Hour) + for _, off := range offsets { + at := base.Add(time.Duration(off) * time.Second) + for maxUser < alphaUsers && !joined[maxUser+1].After(at) { + maxUser++ + } + for readyPairs < len(byReady) && !byReady[readyPairs].ready.After(at) { + readyPairs++ + } var ch, author int64 - if isDM[id] { - p := pairs[skewedIndex(rng, len(pairs))] + if isDM[id] && readyPairs > 0 { + p := byReady[skewedIndex(rng, readyPairs)] ch = p.channelID if rng.Intn(2) == 0 { author = p.a @@ -505,10 +564,11 @@ func alphaInsertMessages(tx *sql.Tx, rng *rand.Rand, windowStart time.Time, pair } } else { ch = pickChannel(day) - author = pickAuthor(ch) + author = pickAuthor(ch, maxUser) } id++ times = append(times, at) + authors = append(authors, author) if len(args) > 0 { b.WriteString(",") } @@ -516,21 +576,21 @@ func alphaInsertMessages(tx *sql.Tx, rng *rand.Rand, windowStart time.Time, pair args = append(args, id, ch, author, sentence(rng), ts(at)) if len(args) >= chunk*cols { if err := flush(); err != nil { - return nil, fmt.Errorf("alpha: messages: %w", err) + return nil, nil, fmt.Errorf("alpha: messages: %w", err) } } } } } if err := flush(); err != nil { - return nil, fmt.Errorf("alpha: messages: %w", err) + return nil, nil, fmt.Errorf("alpha: messages: %w", err) } - return times, nil + return times, authors, nil } // ─── Attachments, reactions, invites ──────────────────────────────────────── -func alphaInsertAttachments(tx *sql.Tx, rng *rand.Rand, messageTimes []time.Time) error { +func alphaInsertAttachments(tx *sql.Tx, rng *rand.Rand, messageTimes []time.Time, messageAuthors []int64) error { type class struct { count int mime, ext string @@ -550,12 +610,14 @@ func alphaInsertAttachments(tx *sql.Tx, rng *rand.Rand, messageTimes []time.Time msgID := rng.Intn(alphaMessages) + 1 id := fmt.Sprintf("%08x%08x%08x%08x", rng.Uint32(), rng.Uint32(), rng.Uint32(), rng.Uint32()) size := (c.minKB + rng.Intn(c.maxKB-c.minKB+1)) * 1024 + // The uploader is the message's author — CreateAttachment always + // records one on the current schema (Codex on #1469, P2). if _, err := tx.Exec( - `INSERT INTO attachments (id, message_id, filename, stored_as, mime_type, size, uploaded_at) - VALUES (?, ?, ?, ?, ?, ?, ?)`, + `INSERT INTO attachments (id, message_id, filename, stored_as, mime_type, size, uploaded_at, uploader_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, id, msgID, fmt.Sprintf("%s-%03d.%s", c.namePrefix, n, c.ext), - id+"."+c.ext, c.mime, size, ts(messageTimes[msgID-1]), + id+"."+c.ext, c.mime, size, ts(messageTimes[msgID-1]), messageAuthors[msgID-1], ); err != nil { return fmt.Errorf("alpha: attachment %d: %w", n, err) } diff --git a/Server/db/alpha_snapshot_test.go b/Server/db/alpha_snapshot_test.go index 380bd541..63e8182e 100644 --- a/Server/db/alpha_snapshot_test.go +++ b/Server/db/alpha_snapshot_test.go @@ -110,4 +110,26 @@ func TestAlphaSnapshotMigratesOnHead(t *testing.T) { if got := count(`SELECT COUNT(*) FROM messages_fts`); got != 20000 { t.Errorf("messages_fts holds %d rows, want 20000", got) } + + // Realism invariants a live database cannot violate (Codex on #1469): + // nothing is authored before its author joined, ids follow timestamps as + // insertion order does, and every attachment records its uploader. + for _, tc := range []struct { + what, q string + }{ + {"messages authored before their author joined", + `SELECT COUNT(*) FROM messages m JOIN users u ON u.id = m.user_id WHERE m.timestamp < u.created_at`}, + {"adjacent message ids with inverted timestamps", + `SELECT COUNT(*) FROM messages a JOIN messages b ON b.id = a.id + 1 WHERE b.timestamp < a.timestamp`}, + {"attachments without an uploader", + `SELECT COUNT(*) FROM attachments WHERE uploader_id IS NULL`}, + {"DM messages before both members existed", + `SELECT COUNT(*) FROM messages m JOIN channels c ON c.id = m.channel_id AND c.type = 'dm' + JOIN dm_participants dp ON dp.channel_id = c.id JOIN users u ON u.id = dp.user_id + WHERE m.timestamp < u.created_at`}, + } { + if got := count(tc.q); got != 0 { + t.Errorf("%s: %d, want 0", tc.what, got) + } + } } diff --git a/Server/invariants/db_import_boundary.go b/Server/invariants/db_import_boundary.go index 725d6bf7..d51fe4d3 100644 --- a/Server/invariants/db_import_boundary.go +++ b/Server/invariants/db_import_boundary.go @@ -80,6 +80,7 @@ var DBImportAllow = map[string]DBImportEntry{ "internal/app/plugins.go": {"boundary", "", "passes the handle to the plugin registry as its store; no calls"}, "token_cli.go": {"move", "auth", "API-token CLI duplicates admin/handlers_tokens.go"}, "cmd/seed/main.go": {"boundary", "", "developer seeding tool owns its handle"}, + "cmd/seed/profile_alpha.go": {"boundary", "", "the alpha profile writes through the handle main.go owns"}, "cmd/gendocs/main.go": {"boundary", "", "docs generator migrates its own in-memory catalog"}, "plugin/pluginstore.go": {"adapter", "", "PluginRow type only; the store is injected"}, // ── ws ──────────────────────────────────────────────────────────────── diff --git a/Server/testdata/snapshots/README.md b/Server/testdata/snapshots/README.md index defa1fbb..688b467a 100644 --- a/Server/testdata/snapshots/README.md +++ b/Server/testdata/snapshots/README.md @@ -51,6 +51,10 @@ the snapshot stops being an alpha.4 artifact, `alphaSnapshotMigrations` in the canary must move with it, and the change belongs in the B3-7 evidence block. -`scrub.sql` is the identity scrub, written so it can also anonymise a real -donated alpha database: usernames, profiles, secrets, sessions, tokens, -invite codes, audit detail, filenames and wall-clock migration timestamps. +`scrub.sql` is the identity and credential scrub: usernames, profiles, +secrets, sessions, tokens, invite codes, audit detail, filenames and +wall-clock migration timestamps. It deliberately does not touch message +content or channel names — synthetic here, but a judgement call on a real +database — so a donated production database is **not** shareable after this +script alone: clear or rewrite its content (and rebuild FTS) first, or +refuse the donation. diff --git a/Server/testdata/snapshots/scrub.sql b/Server/testdata/snapshots/scrub.sql index 52ad0e97..0b586195 100644 --- a/Server/testdata/snapshots/scrub.sql +++ b/Server/testdata/snapshots/scrub.sql @@ -1,12 +1,19 @@ --- Identity scrub for alpha snapshots (B3-7 item 2). +-- Identity and credential scrub for alpha snapshots (B3-7 item 2). -- -- Applied by `go run ./cmd/seed -profile alpha -snapshot … -scrub this-file` --- before VACUUM INTO writes the committed snapshot, and equally applicable to --- a REAL alpha database an operator wants to donate as a test fixture: every --- statement is idempotent, and together they remove or replace everything --- that identifies a person or authenticates a session. Statements are split --- on ";" by the seed tool, so keep one statement per block and comments on --- their own lines. +-- before VACUUM INTO writes the committed snapshot. Every statement is +-- idempotent. Scope, stated precisely (Codex on #1469): this script removes +-- account identities, credentials, session/token material, invite codes, +-- audit detail, user-chosen filenames and wall-clock apply times. It +-- DELIBERATELY DOES NOT touch message content, channel names/topics, or the +-- FTS index built from them — the committed snapshot's content is synthetic +-- lexicon text with nothing to hide, and content anonymisation of a REAL +-- database is a judgement call no blanket UPDATE can make. A donated +-- production database is NOT made shareable by this script alone: its +-- messages and channel names must be cleared or rewritten first (and the +-- FTS index rebuilt), or the donation refused. Statements are split on ";" +-- with comment lines stripped, so keep one statement per block and comments +-- on their own lines. UPDATE users SET username = 'user' || printf('%03d', id), diff --git a/Server/testdata/snapshots/v1.2.0-alpha.4.sqlite b/Server/testdata/snapshots/v1.2.0-alpha.4.sqlite index eca0ec74..0eb0fe66 100644 Binary files a/Server/testdata/snapshots/v1.2.0-alpha.4.sqlite and b/Server/testdata/snapshots/v1.2.0-alpha.4.sqlite differ diff --git a/docs/architecture/server-boundaries.md b/docs/architecture/server-boundaries.md index cd29f0f2..26d6ff41 100644 --- a/docs/architecture/server-boundaries.md +++ b/docs/architecture/server-boundaries.md @@ -91,6 +91,7 @@ which is a row worth reading, and none exists today. | `auth/resolve.go` | `APIToken` `Role×2` `Session×2` `User×2` | — | — | type-only | adapter | — | Session/APIToken/Role/User types; resolution is injected | | `cmd/gendocs/main.go` | `DB×3` | `Migrate()` `Open()` | `Close×2` `QueryContext×2` | calls | boundary | — | docs generator migrates its own in-memory catalog | | `cmd/seed/main.go` | `DB×6` | `Migrate()` `Open()` | `Close` `CreateChannel` `CreateMessage×2` `CreateUser` `GetOrCreateDMChannel` `GetUserByUsername` `ListChannels` `QueryRowContext` | calls | boundary | — | developer seeding tool owns its handle | +| `cmd/seed/profile_alpha.go` | `DB×3` | `Migrate()` | `BeginTx` `ExecContext×2` `QueryRowContext` | calls | boundary | — | the alpha profile writes through the handle main.go owns | | `internal/app/app.go` | `AuditWriter` `DB` | — | — | type-only | boundary | — | the App holds the handle for its lifetime; no calls | | `internal/app/database.go` | `DB×2` | `Migrate()` `OpenWithMaxReaders()` | `ClearAllVoiceStates` `ResetAllUserStatuses` | calls | boundary | — | opens the handle, migrates, clears stale state at boot | | `internal/app/hub.go` | `DB` | — | — | type-only | boundary | — | hands the handle to the hub and the service layer it builds | @@ -117,8 +118,8 @@ which is a row worth reading, and none exists today. | `ws/voice_join.go` | `Channel×3` `ChannelOverride` `VoiceState×5` | `ErrChannelFull` | `GetChannel×2` `GetChannelOverridesFor` `GetChannelVoiceStates` `GetRoleForUser` `GetVoiceState×6` `JoinVoiceChannel` `JoinVoiceChannelIfCapacity` `LeaveVoiceChannelIfMatch` `SetVoiceServerDeafen` `SetVoiceServerMute` | calls | move | voice | voice state reads and writes | | `ws/voice_moderation.go` | `Role` `VoiceState×3` | `WriteAudit()` | `CountChannelVoiceUsers` `GetChannel×2` `GetRoleForUser` `GetVoiceState×2` `SetVoiceServerDeafen×2` `SetVoiceServerMute×2` | calls | move | voice | mute/deafen/move persist voice state | -55 files import `db` outside `db/` and `service/` (. 1, admin 16, api 10, auth 2, cmd/gendocs 1, cmd/seed 1, internal/app 6, plugin 1, ws 17); 17 are type-only; 0 unlisted. -Dispositions: adapter 17, boundary 12, move 26. Move targets: auth 7, channel 6, connection 1, role 1, settings-ops 4, upload 2, user 2, voice 3. +56 files import `db` outside `db/` and `service/` (. 1, admin 16, api 10, auth 2, cmd/gendocs 1, cmd/seed 2, internal/app 6, plugin 1, ws 17); 17 are type-only; 0 unlisted. +Dispositions: adapter 17, boundary 13, move 26. Move targets: auth 7, channel 6, connection 1, role 1, settings-ops 4, upload 2, user 2, voice 3.