From 3c2c83e6373a7beac8d2efc00d043276c714c613 Mon Sep 17 00:00:00 2001 From: scher Date: Thu, 3 Sep 2026 20:04:44 +0300 Subject: [PATCH] drop the transport from a location key --- index/index.v | 74 ++++++++++++++++++++++++++++++++++---------- index/schema.v | 10 +++++- source/source.v | 21 +++++++++++-- source/source_test.v | 7 ++++- 4 files changed, 91 insertions(+), 21 deletions(-) diff --git a/index/index.v b/index/index.v index fd1c9ca..2b33599 100644 --- a/index/index.v +++ b/index/index.v @@ -55,6 +55,12 @@ fn (mut d DB) migrate() ! { return err } } + if v + 1 == 3 { + d.fold_transport_keys() or { + d.conn.rollback() or {} + return err + } + } } d.conn.exec('PRAGMA user_version = ${schema_version}') or { d.conn.rollback() or {} @@ -98,8 +104,7 @@ pub fn (mut d DB) deactivate_sources_except(ids []string) ! { pub fn (mut d DB) resolve_repository(locations []Location, name string, object_format string, now i64) !i64 { mut owners := []i64{} for location in locations { - rows := d.conn.exec_param('SELECT repository_id FROM repository_locations WHERE key = ?', - location.key)! + rows := d.conn.exec_param('SELECT repository_id FROM repository_locations WHERE key = ?', location.key)! if rows.len == 0 { continue } @@ -159,6 +164,50 @@ fn (mut d DB) merge_repositories(keep i64, absorbed i64) ! { d.conn.exec('DELETE FROM repositories WHERE id = ${absorbed}')! } +// fold_transport_keys is the v3 migration. A remote key used to carry the +// transport, so one repository reached over ssh and over https was two rows, +// two repositories and two histories. Rows whose keys fold onto each other are +// merged into the oldest of them. +// +// Every remaining remote row loses its digest and is walked again on the next +// sync. A repository that just absorbed another has a different set of commits +// than the digest was taken for and one rescan is the safe direction to be +// wrong in. +fn (mut d DB) fold_transport_keys() ! { + rows := d.conn.exec("SELECT id, key FROM repository_locations + WHERE kind = 'remote' ORDER BY id")! + mut keeper := map[string]i64{} + for row in rows { + id := row.val(0).i64() + folded := source.fold_transport(row.val(1)) + if folded !in keeper { + keeper[folded] = id + continue + } + // Read both repositories now rather than trusting what they were at the + // top of the loop: an earlier merge may have moved either of them. + keep := d.repository_of_location(keeper[folded])! + absorbed := d.repository_of_location(id)! + if keep != 0 && absorbed != 0 && keep != absorbed { + d.merge_repositories(keep, absorbed)! + } + d.conn.exec('DELETE FROM repository_locations WHERE id = ${id}')! + } + for folded, id in keeper { + d.conn.exec_param_many("UPDATE repository_locations SET key = ?, refs_digest = ''\n\t\t\tWHERE id = ${id}", [ + folded, + ])! + } +} + +fn (mut d DB) repository_of_location(id i64) !i64 { + rows := d.conn.exec('SELECT repository_id FROM repository_locations WHERE id = ${id}')! + if rows.len == 0 { + return 0 + } + return rows[0].val(0).i64() +} + // location_digest is the fingerprint of the refs seen the last time this location // was scanned successfully. pub fn (mut d DB) location_digest(key string) !string { @@ -183,13 +232,11 @@ pub fn (mut d DB) replace_remotes(repository_id i64, remotes map[string]string) d.conn.exec_param('DELETE FROM repository_remotes WHERE repository_id = ?', repository_id.str())! mut rows := [][]string{} for remote_name, url in remotes { - rows << [repository_id.str(), remote_name, source.redact_url(url), - source.normalize_url(url)] + rows << [repository_id.str(), remote_name, source.redact_url(url), source.normalize_url(url)] } if rows.len > 0 { d.conn.exec_param_many('INSERT OR IGNORE INTO repository_remotes (repository_id, name, url, url_norm) - VALUES (?, ?, ?, ?)', - rows)! + VALUES (?, ?, ?, ?)', rows)! } } @@ -252,8 +299,7 @@ fn (mut d DB) apply_accepted(names []string, emails []string) ! { } if rows.len > 0 { d.conn.exec_param_many('INSERT OR IGNORE INTO accepted_identities (kind, value, value_norm) - VALUES (?, ?, ?)', - rows)! + VALUES (?, ?, ?)', rows)! } } @@ -285,8 +331,7 @@ fn (mut d DB) apply_snapshot(repository_id i64, scan gitrepo.Scan, fresh bool) ! } if idents.len > 0 { d.conn.exec_param_many('INSERT OR IGNORE INTO git_identities (name, email, email_norm) - VALUES (?, ?, ?)', - idents)! + VALUES (?, ?, ?)', idents)! } mut rows := [][]string{cap: scan.commits.len} @@ -317,15 +362,13 @@ fn (mut d DB) apply_snapshot(repository_id i64, scan gitrepo.Scan, fresh bool) ! VALUES (?, ?, ?, (SELECT id FROM git_identities WHERE name = ? AND email = ?), ?, ?, date(?, 'unixepoch'), (SELECT id FROM git_identities WHERE name = ? AND email = ?), ?, ?, date(?, 'unixepoch'), - ?)", - rows)! + ?)", rows)! } // Full snapshot replacement. Membership reflects the latest successful scan, // and a deleted branch stops counting with no drift to reconcile. if fresh { - d.conn.exec_param('DELETE FROM repository_commits WHERE repository_id = ?', - repository_id.str())! + d.conn.exec_param('DELETE FROM repository_commits WHERE repository_id = ?', repository_id.str())! } mut members := [][]string{cap: scan.commits.len} for c in scan.commits { @@ -333,8 +376,7 @@ fn (mut d DB) apply_snapshot(repository_id i64, scan gitrepo.Scan, fresh bool) ! } if members.len > 0 { d.conn.exec_param_many('INSERT OR IGNORE INTO repository_commits (repository_id, commit_id) - VALUES (?, (SELECT id FROM commits WHERE object_format = ? AND object_id = ?))', - members)! + VALUES (?, (SELECT id FROM commits WHERE object_format = ? AND object_id = ?))', members)! } d.conn.exec_param_many('UPDATE repositories SET object_format = ? WHERE id = ?', [ diff --git a/index/schema.v b/index/schema.v index 5b4757e..ecb40d9 100644 --- a/index/schema.v +++ b/index/schema.v @@ -2,7 +2,7 @@ module index // schema_version is bumped whenever a migration is appended. A database written // by a newer gitlife is refused rather than guessed at. -const schema_version = 2 +const schema_version = 3 // migrations[v - 1] holds the statements that take the schema to version v. // Statements are listed one per entry because SQLite prepares a single statement @@ -10,6 +10,7 @@ const schema_version = 2 const migrations = [ v1, v2, + v3, ] const v1 = [ @@ -107,3 +108,10 @@ const v2 = [ "ALTER TABLE repository_locations ADD COLUMN refs_digest TEXT NOT NULL DEFAULT ''", 'ALTER TABLE repositories DROP COLUMN refs_digest', ] + +// v3 drops the transport from a remote location key which used to keep +// 'ssh://host/you/repo' and 'https://host/you/repo' apart as two repositories. +// It has no statements: keys collide once they fold and the rows that collide +// can belong to two repository rows that have to be merged first. That work is +// fold_transport_keys in index.v. +const v3 = []string{} diff --git a/source/source.v b/source/source.v index c75333d..712a9a0 100644 --- a/source/source.v +++ b/source/source.v @@ -40,7 +40,11 @@ pub fn location_key(raw string) string { if path := local_path(raw) { return 'dir:' + os.real_path(path) } - return 'url:' + normalize_url(raw) + normalized := normalize_url(raw) + if at := normalized.index('://') { + return 'url:' + normalized[at + 3..] + } + return 'url:' + normalized } // local_path recognizes the addresses that name a directory on this machine @@ -53,14 +57,25 @@ pub fn local_path(raw string) ?string { if address.contains('://') { return none } - if address.starts_with('/') || address.starts_with('./') || address.starts_with('../') - || address.starts_with('~/') { + if address.starts_with('/') || address.starts_with('./') || address.starts_with('../') || address.starts_with('~/') { return address } // Anything else is a remote: 'host:path', 'user@host:path', a bare name. return none } +// fold_transport rewrites a location key that was stored before the transport +// was dropped from one. Only the v3 migration in index has any use for it. +pub fn fold_transport(key string) string { + if !key.starts_with('url:') { + return key + } + if at := key.index('://') { + return 'url:' + key[at + 3..] + } + return key +} + pub fn redact_url(raw string) string { scheme_end := raw.index('://') or { return raw } rest := raw[scheme_end + 3..] diff --git a/source/source_test.v b/source/source_test.v index 9faeff5..9c112cf 100644 --- a/source/source_test.v +++ b/source/source_test.v @@ -39,7 +39,12 @@ fn test_a_fork_and_its_upstream_do_not_collapse() { } fn test_the_key_is_namespaced() { - assert location_key('https://github.com/a/b') == 'url:https://github.com/a/b' + assert location_key('https://github.com/a/b') == 'url:github.com/a/b' +} + +fn test_ssh_and_https_spellings_share_a_key() { + assert location_key('git@github.com:nepinhum/gitlife.git') == location_key('https://github.com/nepinhum/gitlife') + assert location_key('ssh://git@github.com/a/b') == location_key('https://github.com/a/b') } fn test_display_name_is_owner_and_repository() {