From 3d3a91044f22a3d9310afcc4e2a5c9f44131a973 Mon Sep 17 00:00:00 2001 From: scher Date: Sat, 5 Sep 2026 02:24:44 +0300 Subject: [PATCH 01/11] up: walk a known repository as a difference --- gitrepo/README.md | 10 +++ gitrepo/gitrepo.v | 121 +++++++++++++++++++++++++++++++----- index/README.md | 6 ++ index/index.v | 155 ++++++++++++++++++++++++++++++++++------------ index/schema.v | 11 +++- syncer/README.md | 8 ++- syncer/syncer.v | 90 ++++++++++++++++++++++----- 7 files changed, 325 insertions(+), 76 deletions(-) diff --git a/gitrepo/README.md b/gitrepo/README.md index e97131f..2ed7c57 100644 --- a/gitrepo/README.md +++ b/gitrepo/README.md @@ -8,3 +8,13 @@ Records are read with a fixed arity of eleven lines per commit rather than a NUL-delimited format because V's process helpers truncate at the first NUL. Every calendar facing value keeps the commit's own timezone offset, which makes a day the day the author experienced rather than the day it was in UTC. + +A repository the index has seen before is walked as a difference. `refs` keeps +the tips a walk stopped at and the next one asks git for the commits that came +into scope since (`added`) and the ones that left it (`dropped`) rather than for +the whole history again. Both fail when a tip is no longer in the repository, +which is what a rewrite followed by a `git gc` leaves behind and the caller +walks everything instead. + +Every walk shares one ref scope. An incremental walk that disagreed with the full +one about which refs count would add commits that a later full walk removes. diff --git a/gitrepo/gitrepo.v b/gitrepo/gitrepo.v index 92f6748..a9f57a5 100644 --- a/gitrepo/gitrepo.v +++ b/gitrepo/gitrepo.v @@ -11,6 +11,11 @@ const commit_format = '%H%n%P%n%an%n%ae%n%at%n%az%n%cn%n%ce%n%ct%n%cz%n%s' const commit_fields = 11 +// ref_scope is what a scan considers reachable. Every walk uses the same +// scope or an incremental one would disagree with the full one about what a +// repository holds. +const ref_scope = ['--exclude=refs/stash', '--exclude=refs/notes/*', '--all'] + pub struct Commit { pub: object_id string @@ -26,6 +31,14 @@ pub: subject string } +// Refs is where a location's refs stood at a scan: the fingerprint that says +// whether anything moved and the tips a later scan walks from. +pub struct Refs { +pub: + digest string + tips []string +} + pub struct Scan { pub: object_format string @@ -70,12 +83,21 @@ pub fn (g Git) object_format(dir string) !string { return f } -// refs_digest fingerprints every ref plus HEAD. An unchanged digest means no ref -// moved, meaning there is nothing to walk. -pub fn (g Git) refs_digest(dir string) !string { +// refs fingerprints every ref plus HEAD and keeps the tips inside the scan's +// scope. An unchanged digest means no ref moved, meaning there is nothing to +// walk. The tips are what the next scan asks git to walk from. +pub fn (g Git) refs(dir string) !Refs { r := g.at(dir, ['for-each-ref', '--format=%(objectname) %(refname)'])! mut lines := r.stdout.split_into_lines().filter(it != '') lines.sort() + mut tips := []string{} + mut seen := map[string]bool{} + for line in lines { + space := line.index(' ') or { continue } + if in_scope(line[space + 1..]) { + add_tip(mut tips, mut seen, line[..space]) + } + } // An empty repository has no HEAD to resolve; that is not a failure. head := proc.run(proc.Cmd{ exe: g.exe @@ -83,8 +105,27 @@ pub fn (g Git) refs_digest(dir string) !string { cwd: dir env: git_env(map[string]string{}) })! - lines << 'HEAD ' + head.stdout.trim_space() - return sha256.sum256(lines.join('\n').bytes()).hex() + oid := head.stdout.trim_space() + add_tip(mut tips, mut seen, oid) + lines << 'HEAD ' + oid + return Refs{ + digest: sha256.sum256(lines.join('\n').bytes()).hex() + tips: tips + } +} + +// in_scope mirrors ref_scope which git applies and we can't ask it about. +fn in_scope(name string) bool { + return name != 'refs/stash' && !name.starts_with('refs/notes/') +} + +// add_tip keeps one entry per object. Ten branches on one commit are one tip. +fn add_tip(mut tips []string, mut seen map[string]bool, oid string) { + if oid == '' || oid in seen { + return + } + seen[oid] = true + tips << oid } // remotes lists a repository's remotes by name. @@ -125,28 +166,74 @@ pub fn primary_remote(remotes map[string]string) string { return '' } -// commits walks every ref. refs/stash and refs/notes are excluded: they are -// scratch space and metadata, not a record of work. -// -// The log is read as it arrives. A repository with a million commits would -// otherwise exist three times over: git's output as one string, that string cut -// into lines, and the commits themselves. +// commits walks everything in scope. The log is read as it arrives: a repository +// with a million commits would otherwise exist three times over, as git's output +// in one string, as that string cut into lines and as the commits themselves. pub fn (g Git) commits(dir string) ![]Commit { + return g.log(dir, [])! +} + +// added walks the commits that came into scope since a scan whose tips these +// were. Everything the old tips reach is indexed already, so git is asked to +// mark it uninteresting and print the rest. +// +// It fails when a tip is no longer in the repository. +pub fn (g Git) added(dir string, previous []string) ![]Commit { + mut exclude := ['--not'] + exclude << previous + return g.log(dir, exclude)! +} + +// dropped names the commits the old tips reached that nothing reaches now: a +// deleted branch or history rewritten out from under one. Only the object ids, +// since these are commits the index already holds and is about to stop counting +// for this repository. +pub fn (g Git) dropped(dir string, previous []string) ![]string { + mut args := ['rev-list'] + args << previous + args << '--not' + args << ref_scope + mut oids := Oids{} + proc.check_stream(proc.Cmd{ + exe: g.exe + args: args + cwd: dir + env: git_env(map[string]string{}) + }, mut oids)! + return oids.list +} + +fn (g Git) log(dir string, extra []string) ![]Commit { + mut args := ['log'] + args << ref_scope + args << ['--no-use-mailmap', '--no-show-signature', '--format=' + commit_format] + args << extra mut reader := CommitReader{} proc.check_stream(proc.Cmd{ - exe: g.exe - args: ['log', '--exclude=refs/stash', '--exclude=refs/notes/*', '--all', '--no-use-mailmap', - '--no-show-signature', '--format=' + commit_format] - cwd: dir - env: git_env(map[string]string{}) + exe: g.exe + args: args + cwd: dir + env: git_env(map[string]string{}) }, mut reader)! return reader.done() } +// An Oids collects a walk that prints one object id per line. +struct Oids { +mut: + list []string +} + +fn (mut o Oids) take(line string) ! { + if line != '' { + o.list << line + } +} + pub fn (g Git) scan(dir string) !Scan { return Scan{ object_format: g.object_format(dir)! - refs_digest: g.refs_digest(dir)! + refs_digest: g.refs(dir)!.digest commits: g.commits(dir)! } } diff --git a/index/README.md b/index/README.md index 98efbe3..a74c34d 100644 --- a/index/README.md +++ b/index/README.md @@ -26,3 +26,9 @@ two reports from disagreeing about what belongs to the user. `purge` forgets what no configured source can still reach. It is deliberately a separate act from removing a source. `--dry-run` is the same code path with the transaction rolled back instead of committed. + +`write_delta` is the other way in: the commits a location gained and the ones it +lost for a repository whose membership can only have come from that one +location. A repository reached through several locations holds the union of them +and one location losing sight of a commit says nothing about the others, so those +are written as whole snapshots. diff --git a/index/index.v b/index/index.v index 651aaad..67a7372 100644 --- a/index/index.v +++ b/index/index.v @@ -212,23 +212,43 @@ fn (mut d DB) repository_of_location(id i64) !i64 { return rows[0].val(0).i64() } -// location_digest is the fingerprint of the refs seen the last time this location -// was scanned successfully. -pub fn (mut d DB) location_digest(key string) !string { - rows := d.conn.exec_param('SELECT refs_digest FROM repository_locations WHERE key = ?', key)! +// LocationState is what the last successful scan of a location left behind: the +// fingerprint of the refs it saw and the tips it stopped at. +pub struct LocationState { +pub: + digest string + tips []string +} + +pub fn (mut d DB) location_state(key string) !LocationState { + rows := d.conn.exec_param('SELECT refs_digest, ref_tips FROM repository_locations WHERE key = ?', + key)! if rows.len == 0 { - return '' + return LocationState{} + } + return LocationState{ + digest: rows[0].val(0) + tips: rows[0].val(1).split(' ').filter(it != '') } - return rows[0].val(0) } -pub fn (mut d DB) set_location_digest(key string, digest string) ! { - d.conn.exec_param_many('UPDATE repository_locations SET refs_digest = ? WHERE key = ?', [ +pub fn (mut d DB) set_location_state(key string, digest string, tips []string) ! { + d.conn.exec_param_many('UPDATE repository_locations SET refs_digest = ?, ref_tips = ? + WHERE key = ?', [ digest, + tips.join(' '), key, ])! } +// scanned_locations counts the locations of a repository that have ever been +// walked. Membership is the union of them, a location can only reason about +// what a repository holds on its own when it is the only one. +pub fn (mut d DB) scanned_locations(repository_id i64) !int { + return d.conn.q_int("SELECT count(*) FROM repository_locations + WHERE repository_id = ${repository_id} AND refs_digest != ''")! +} + // replace_remotes records every remote a repository has, as evidence. These are // deliberately not locations: a non primary remote must never merge two // repositories or adding 'upstream' to a fork would fuse it with what it forked. @@ -327,18 +347,99 @@ pub fn (mut d DB) write_snapshot(repository_id i64, scan gitrepo.Scan, fresh boo fn (mut d DB) apply_snapshot(repository_id i64, scan gitrepo.Scan, fresh bool) !int { before := d.conn.q_int('SELECT COALESCE(MAX(id), 0) FROM commits')! + d.insert_commits(scan.object_format, scan.commits)! + + // Membership is diffed against what the location holds, not replaced by it. + d.conn.exec('DELETE FROM scanned_commits')! + mut scanned := Batch{ + query: 'INSERT OR IGNORE INTO scanned_commits (commit_id) + SELECT id FROM commits WHERE object_format = ? AND object_id = ?' + } + for c in scan.commits { + scanned.add(mut d, [scan.object_format, c.object_id])! + } + scanned.send(mut d)! + + id := repository_id.str() + d.conn.exec_param('INSERT OR IGNORE INTO repository_commits (repository_id, commit_id) + SELECT ?, commit_id FROM scanned_commits', id)! + if fresh { + d.conn.exec_param('DELETE FROM repository_commits + WHERE repository_id = ? + AND commit_id NOT IN (SELECT commit_id FROM scanned_commits)', id)! + } + + d.conn.exec_param_many('UPDATE repositories SET object_format = ? WHERE id = ?', [ + scan.object_format, + id, + ])! + return int(d.conn.q_int('SELECT COALESCE(MAX(id), 0) FROM commits')! - before) +} + +// write_delta records what changed for a location instead of what it holds: +// the commits that came into scope and the ones that left it. It returns how +// many commits the index had never seen and how many the repository holds now. +// +// Only a repository with a single scanned location can be written this way. A +// commit leaving one location's scope says nothing about whether another +// location still reaches it, and membership is the union of them. +pub fn (mut d DB) write_delta(repository_id i64, object_format string, added []gitrepo.Commit, dropped []string) !(int, int) { + d.conn.begin()! + fresh, held := d.apply_delta(repository_id, object_format, added, dropped) or { + d.conn.rollback() or {} + return err + } + d.conn.commit()! + return fresh, held +} + +fn (mut d DB) apply_delta(repository_id i64, object_format string, added []gitrepo.Commit, dropped []string) !(int, int) { + before := d.conn.q_int('SELECT COALESCE(MAX(id), 0) FROM commits')! + d.insert_commits(object_format, added)! + + id := repository_id.str() + mut members := Batch{ + query: 'INSERT OR IGNORE INTO repository_commits (repository_id, commit_id) + VALUES (?, (SELECT id FROM commits WHERE object_format = ? AND object_id = ?))' + } + for c in added { + members.add(mut d, [id, object_format, c.object_id])! + } + members.send(mut d)! + + // A commit nothing reaches any more stops counting for this repository. The + // commit row itself stays: another repository may hold it and purge is what + // forgets a commit for good. + mut gone := Batch{ + query: 'DELETE FROM repository_commits + WHERE repository_id = ? + AND commit_id = (SELECT id FROM commits WHERE object_format = ? AND object_id = ?)' + } + for oid in dropped { + gone.add(mut d, [id, object_format, oid])! + } + gone.send(mut d)! + + held := d.conn.q_int('SELECT count(*) FROM repository_commits WHERE repository_id = ${repository_id}')! + return int(d.conn.q_int('SELECT COALESCE(MAX(id), 0) FROM commits')! - before), held +} + +// insert_commits writes the commits themselves and the identities they name. +// A commit row is never updated, so the same commit reached twice costs a lookup +// and nothing else. +fn (mut d DB) insert_commits(object_format string, commits []gitrepo.Commit) ! { mut idents := Batch{ query: 'INSERT OR IGNORE INTO git_identities (name, email, email_norm) VALUES (?, ?, ?)' } mut seen := map[string]bool{} - for c in scan.commits { + for c in commits { add_identity(mut idents, mut d, mut seen, c.author_name, c.author_email)! add_identity(mut idents, mut d, mut seen, c.committer_name, c.committer_email)! } idents.send(mut d)! - mut commits := Batch{ + mut rows := Batch{ query: "INSERT OR IGNORE INTO commits ( object_format, object_id, parents, author_identity_id, author_time, author_tz, author_date, @@ -349,9 +450,9 @@ fn (mut d DB) apply_snapshot(repository_id i64, scan gitrepo.Scan, fresh bool) ! (SELECT id FROM git_identities WHERE name = ? AND email = ?), ?, ?, date(?, 'unixepoch'), ?)" } - for c in scan.commits { - commits.add(mut d, [ - scan.object_format, + for c in commits { + rows.add(mut d, [ + object_format, c.object_id, c.parents, c.author_name, @@ -367,33 +468,7 @@ fn (mut d DB) apply_snapshot(repository_id i64, scan gitrepo.Scan, fresh bool) ! c.subject, ])! } - commits.send(mut d)! - - // Membership is diffed against what the location holds, not replaced by it. - d.conn.exec('DELETE FROM scanned_commits')! - mut scanned := Batch{ - query: 'INSERT OR IGNORE INTO scanned_commits (commit_id) - SELECT id FROM commits WHERE object_format = ? AND object_id = ?' - } - for c in scan.commits { - scanned.add(mut d, [scan.object_format, c.object_id])! - } - scanned.send(mut d)! - - id := repository_id.str() - d.conn.exec_param('INSERT OR IGNORE INTO repository_commits (repository_id, commit_id) - SELECT ?, commit_id FROM scanned_commits', id)! - if fresh { - d.conn.exec_param('DELETE FROM repository_commits - WHERE repository_id = ? - AND commit_id NOT IN (SELECT commit_id FROM scanned_commits)', id)! - } - - d.conn.exec_param_many('UPDATE repositories SET object_format = ? WHERE id = ?', [ - scan.object_format, - id, - ])! - return int(d.conn.q_int('SELECT COALESCE(MAX(id), 0) FROM commits')! - before) + rows.send(mut d)! } // rows_per_batch is how many rows an insert holds at once. The statement is diff --git a/index/schema.v b/index/schema.v index ecb40d9..d88db11 100644 --- a/index/schema.v +++ b/index/schema.v @@ -2,7 +2,7 @@ module index // schema_version is bumped whenever a migration is appended. A database written // by a newer gitlife is refused rather than guessed at. -const schema_version = 3 +const schema_version = 4 // migrations[v - 1] holds the statements that take the schema to version v. // Statements are listed one per entry because SQLite prepares a single statement @@ -11,6 +11,7 @@ const migrations = [ v1, v2, v3, + v4, ] const v1 = [ @@ -115,3 +116,11 @@ const v2 = [ // can belong to two repository rows that have to be merged first. That work is // fold_transport_keys in index.v. const v3 = []string{} + +// v4 keeps the ref tips a location was last scanned at, next to the digest that +// says whether they moved. A scan that knows where the last one stopped can ask +// git for the difference instead of the whole history. Existing rows start +// empty, costing each location one full walk and nothing after that. +const v4 = [ + "ALTER TABLE repository_locations ADD COLUMN ref_tips TEXT NOT NULL DEFAULT ''", +] diff --git a/syncer/README.md b/syncer/README.md index 492bb6d..a5877d4 100644 --- a/syncer/README.md +++ b/syncer/README.md @@ -11,7 +11,7 @@ database, and no worker thread ever holds the connection. discover what repositories exist (serial, one call per source) prepare clone or fetch, fingerprint refs (parallel, no database) register resolve identity, decide what moved (serial, database) -read walk the commits (parallel, no database) +read walk the commits, or what changed (parallel, no database) write bring the snapshots up to date (serial, database) ``` @@ -21,6 +21,12 @@ held from the moment its worker finished with it until it is in the database, and not a moment longer, so a run costs a history per worker rather than one per repository. +A location the index has walked before is walked as a difference: its stored ref +tips say where the last walk stopped and git is asked for what came into scope +since and what left it. A location seen for the first time, one whose old tips +have been garbage collected and any repository with more than one walked +location are walked whole. + Registration is separate from preparation because a repository can be reachable through more than one location. A working tree and a cached clone of the same origin are one repository and its membership is the union of them, not whichever diff --git a/syncer/syncer.v b/syncer/syncer.v index 4d69feb..dda8b8b 100644 --- a/syncer/syncer.v +++ b/syncer/syncer.v @@ -93,7 +93,7 @@ struct Prepared { action string remotes map[string]string object_format string - digest string + refs gitrepo.Refs elapsed_ms int error string } @@ -106,10 +106,13 @@ struct Task { location_key string dir string object_format string - digest string + refs gitrepo.Refs action string elapsed_ms int // what preparing it already cost changed bool + // previous holds the tips this location was last walked at and is empty + // when the whole history has to be walked instead. See incremental. + previous []string // fresh marks the first location of its repository to be written this run. // Only that one clears the previous membership; the rest add to it. fresh bool @@ -118,7 +121,11 @@ struct Task { // Scanned is a worker's walk of one task's commits. It lives from the moment its // worker hands it over until the writer is done with it and no longer. struct Scanned { - commits []gitrepo.Commit + commits []gitrepo.Commit + // dropped names commits that left the location's scope and delta says + // whether commits is everything the location holds or only what is new. + dropped []string + delta bool elapsed_ms int error string } @@ -321,7 +328,7 @@ fn gather(p Pass, job Job) !Prepared { action: action remotes: remotes object_format: p.git.object_format(dir)! - digest: p.git.refs_digest(dir)! + refs: p.git.refs(dir)! } } @@ -377,6 +384,7 @@ fn enter(item Prepared, mut d index.DB, now i64) !Task { } scanned := locations[0].key + last := d.location_state(scanned)! return Task{ source_id: item.source_id repository_id: repository_id @@ -384,13 +392,30 @@ fn enter(item Prepared, mut d index.DB, now i64) !Task { location_key: scanned dir: item.dir object_format: item.object_format - digest: item.digest + refs: item.refs action: item.action elapsed_ms: item.elapsed_ms - changed: item.digest != d.location_digest(scanned)! + changed: item.refs.digest != last.digest + previous: incremental(last, repository_id, mut d) } } +// incremental decides whether this location can be walked as a difference and +// answers with the tips to walk from or with nothing. +// +// Two conditions. The location must have been walked before or there is no +// difference to take. And it must be the only walked location of its repository: +// membership is the union of a repository's locations. +fn incremental(last index.LocationState, repository_id i64, mut d index.DB) []string { + if last.digest == '' || last.tips.len == 0 { + return []string{} + } + if d.scanned_locations(repository_id) or { 2 } != 1 { + return []string{} + } + return last.tips +} + // plan splits the registered tasks into the ones to walk and the ones to leave // alone. A repository whose locations all still hold the refs they held last time // is not walked at all; if any one of them moved, every location of it is walked @@ -418,9 +443,20 @@ fn plan(tasks []Task) ([]Task, []Task) { return scan, unchanged } -// read walks one location's commits and times the walk. +// read walks one location and times the walk. A location the index has seen +// before is walked as a difference: what came into scope since the last walk and +// what left it. Everything else is walked whole. fn read(p Pass, task Task) Scanned { started := time.ticks() + if task.previous.len > 0 { + if delta := read_delta(p, task) { + return Scanned{ + ...delta + elapsed_ms: int(time.ticks() - started) + } + } + // The old tips are gone + } commits := p.git.commits(task.dir) or { return Scanned{ error: err.msg() @@ -432,6 +468,15 @@ fn read(p Pass, task Task) Scanned { } } +fn read_delta(p Pass, task Task) !Scanned { + dropped := p.git.dropped(task.dir, task.previous)! + return Scanned{ + commits: p.git.added(task.dir, task.previous)! + dropped: dropped + delta: true + } +} + // read_stride is one worker's share, handed over one walk at a time. Same round // robin as prepare_stride, for the same reason. fn read_stride(p Pass, tasks []Task, first int, stride int, out chan Scanned) { @@ -483,22 +528,33 @@ fn store(task Task, scan Scanned, mut d index.DB, mut report Report, mut broken scan.error, now) return } - fresh := d.write_snapshot(task.repository_id, gitrepo.Scan{ - object_format: task.object_format - refs_digest: task.digest - commits: scan.commits - }, task.fresh) or { - fail_repository(mut d, mut report, mut broken, task.source_id, task.name, task.location_key, - err.msg(), now) - return + mut fresh := 0 + mut held := scan.commits.len + if scan.delta { + fresh, held = d.write_delta(task.repository_id, task.object_format, scan.commits, + scan.dropped) or { + fail_repository(mut d, mut report, mut broken, task.source_id, task.name, + task.location_key, err.msg(), now) + return + } + } else { + fresh = d.write_snapshot(task.repository_id, gitrepo.Scan{ + object_format: task.object_format + refs_digest: task.refs.digest + commits: scan.commits + }, task.fresh) or { + fail_repository(mut d, mut report, mut broken, task.source_id, task.name, + task.location_key, err.msg(), now) + return + } } - d.set_location_digest(task.location_key, task.digest) or {} + d.set_location_state(task.location_key, task.refs.digest, task.refs.tips) or {} report.add(Outcome{ source: task.source_id repository: task.name status: 'ok' action: task.action - commits: scan.commits.len + commits: held new_commits: fresh elapsed_ms: task.elapsed_ms + scan.elapsed_ms }) From 5dacbc8ffefa1089c0b37880892ab90fc40f9921 Mon Sep 17 00:00:00 2001 From: scher Date: Sat, 5 Sep 2026 14:42:04 +0300 Subject: [PATCH 02/11] up --- syncer/syncer.v | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/syncer/syncer.v b/syncer/syncer.v index dda8b8b..a4496c6 100644 --- a/syncer/syncer.v +++ b/syncer/syncer.v @@ -124,8 +124,12 @@ struct Scanned { commits []gitrepo.Commit // dropped names commits that left the location's scope and delta says // whether commits is everything the location holds or only what is new. - dropped []string - delta bool + dropped []string + delta bool + // refs is the ref state read again once the walk was done. The walk is only + // allowed to say where the next one may start from when the refs it ended on + // are still the refs it was handed. + refs gitrepo.Refs elapsed_ms int error string } @@ -452,6 +456,7 @@ fn read(p Pass, task Task) Scanned { if delta := read_delta(p, task) { return Scanned{ ...delta + refs: p.git.refs(task.dir) or { gitrepo.Refs{} } elapsed_ms: int(time.ticks() - started) } } @@ -464,6 +469,7 @@ fn read(p Pass, task Task) Scanned { } return Scanned{ commits: commits + refs: p.git.refs(task.dir) or { gitrepo.Refs{} } elapsed_ms: int(time.ticks() - started) } } @@ -548,7 +554,9 @@ fn store(task Task, scan Scanned, mut d index.DB, mut report Report, mut broken return } } - d.set_location_state(task.location_key, task.refs.digest, task.refs.tips) or {} + if scan.refs.digest == task.refs.digest { + d.set_location_state(task.location_key, task.refs.digest, task.refs.tips) or {} + } report.add(Outcome{ source: task.source_id repository: task.name From 4972e5a529ffa179b240501a628bc2a88c878fda Mon Sep 17 00:00:00 2001 From: scher Date: Sat, 5 Sep 2026 14:57:18 +0300 Subject: [PATCH 03/11] improvements --- syncer/syncer.v | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/syncer/syncer.v b/syncer/syncer.v index a4496c6..6e4adf3 100644 --- a/syncer/syncer.v +++ b/syncer/syncer.v @@ -408,8 +408,10 @@ fn enter(item Prepared, mut d index.DB, now i64) !Task { // answers with the tips to walk from or with nothing. // // Two conditions. The location must have been walked before or there is no -// difference to take. And it must be the only walked location of its repository: -// membership is the union of a repository's locations. +// difference to take. And it must be the only location of its repository the +// index has ever walked: membership is the union of a repository's locations. +// Locations that join that union in this run are ruled out in plan which is the +// first place the whole run is known. fn incremental(last index.LocationState, repository_id i64, mut d index.DB) []string { if last.digest == '' || last.tips.len == 0 { return []string{} @@ -437,10 +439,16 @@ fn plan(tasks []Task) ([]Task, []Task) { unchanged << group continue } + // A repository reached through more than one location this run is walked + // whole at every one of them. Its membership is the union of what they + // reach and a difference taken at one location would drop commits the + // others still hold. + alone := group.len == 1 for i, task in group { scan << Task{ ...task - fresh: i == 0 + fresh: i == 0 + previous: if alone { task.previous } else { []string{} } } } } @@ -556,6 +564,8 @@ fn store(task Task, scan Scanned, mut d index.DB, mut report Report, mut broken } if scan.refs.digest == task.refs.digest { d.set_location_state(task.location_key, task.refs.digest, task.refs.tips) or {} + } else { + d.set_location_state(task.location_key, '', []string{}) or {} } report.add(Outcome{ source: task.source_id From e894dde6ad037ca43a70f9976d62b3443f817893 Mon Sep 17 00:00:00 2001 From: scher Date: Sat, 5 Sep 2026 15:21:29 +0300 Subject: [PATCH 04/11] never take a difference from a truncated history --- gitrepo/README.md | 4 +++- gitrepo/gitrepo.v | 15 ++++++++++----- syncer/README.md | 10 ++++++++-- syncer/syncer.v | 23 ++++++++++++++--------- 4 files changed, 35 insertions(+), 17 deletions(-) diff --git a/gitrepo/README.md b/gitrepo/README.md index 2ed7c57..295a173 100644 --- a/gitrepo/README.md +++ b/gitrepo/README.md @@ -14,7 +14,9 @@ the tips a walk stopped at and the next one asks git for the commits that came into scope since (`added`) and the ones that left it (`dropped`) rather than for the whole history again. Both fail when a tip is no longer in the repository, which is what a rewrite followed by a `git gc` leaves behind and the caller -walks everything instead. +walks everything instead. `refs` also says whether the history is truncated: a +fetch can deepen a shallow repository without moving a ref and what it puts in +reach are ancestors of the very tips a difference would exclude. Every walk shares one ref scope. An incremental walk that disagreed with the full one about which refs count would add commits that a later full walk removes. diff --git a/gitrepo/gitrepo.v b/gitrepo/gitrepo.v index a9f57a5..56c4ac9 100644 --- a/gitrepo/gitrepo.v +++ b/gitrepo/gitrepo.v @@ -37,6 +37,10 @@ pub struct Refs { pub: digest string tips []string + // shallow says the history is truncated. What such a repository holds can + // grow without a ref moving, so its digest can't be trusted to say that + // nothing changed. + shallow bool } pub struct Scan { @@ -98,19 +102,20 @@ pub fn (g Git) refs(dir string) !Refs { add_tip(mut tips, mut seen, line[..space]) } } - // An empty repository has no HEAD to resolve; that is not a failure. head := proc.run(proc.Cmd{ exe: g.exe - args: ['rev-parse', '--verify', '--quiet', 'HEAD'] + args: ['rev-parse', '--is-shallow-repository', '--verify', '--quiet', 'HEAD'] cwd: dir env: git_env(map[string]string{}) })! - oid := head.stdout.trim_space() + answers := head.stdout.split_into_lines().filter(it != '') + oid := if answers.len > 1 { answers[1] } else { '' } add_tip(mut tips, mut seen, oid) lines << 'HEAD ' + oid return Refs{ - digest: sha256.sum256(lines.join('\n').bytes()).hex() - tips: tips + digest: sha256.sum256(lines.join('\n').bytes()).hex() + tips: tips + shallow: answers.len > 0 && answers[0] == 'true' } } diff --git a/syncer/README.md b/syncer/README.md index a5877d4..cbe56a9 100644 --- a/syncer/README.md +++ b/syncer/README.md @@ -24,8 +24,14 @@ repository. A location the index has walked before is walked as a difference: its stored ref tips say where the last walk stopped and git is asked for what came into scope since and what left it. A location seen for the first time, one whose old tips -have been garbage collected and any repository with more than one walked -location are walked whole. +have been garbage collected, any repository with more than one walked location +and any repository whose history is truncated are walked whole. + +A walk records where it stopped only when the refs it was handed are still the +refs it ended on. A repository that moved while it was being read was walked at +a state nobody fingerprinted, its location is marked unwalked and the next +sync walks it whole rather than trusting a fingerprint of a history it never +wrote. Registration is separate from preparation because a repository can be reachable through more than one location. A working tree and a cached clone of the same diff --git a/syncer/syncer.v b/syncer/syncer.v index 6e4adf3..ad2f7f9 100644 --- a/syncer/syncer.v +++ b/syncer/syncer.v @@ -399,21 +399,26 @@ fn enter(item Prepared, mut d index.DB, now i64) !Task { refs: item.refs action: item.action elapsed_ms: item.elapsed_ms - changed: item.refs.digest != last.digest - previous: incremental(last, repository_id, mut d) + // A shallow repository is never called unchanged. A fetch can deepen it + // without moving a ref, leaving the digest saying nothing happened + // while the history it holds has grown. + changed: item.refs.digest != last.digest || item.refs.shallow + previous: incremental(last, item.refs, repository_id, mut d) } } // incremental decides whether this location can be walked as a difference and // answers with the tips to walk from or with nothing. // -// Two conditions. The location must have been walked before or there is no -// difference to take. And it must be the only location of its repository the -// index has ever walked: membership is the union of a repository's locations. -// Locations that join that union in this run are ruled out in plan which is the -// first place the whole run is known. -fn incremental(last index.LocationState, repository_id i64, mut d index.DB) []string { - if last.digest == '' || last.tips.len == 0 { +// Three conditions. The location must have been walked before or there is no +// difference to take. It must be the only location of its repository the index +// has ever walked: membership is the union of a repository's locations and +// locations that join that union in this run are ruled out in plan which is the +// first place the whole run is known. And the history must be whole: deepening a +// shallow repository puts ancestors of the old tips in reach and a difference +// taken from those tips calls exactly those ancestors uninteresting. +fn incremental(last index.LocationState, refs gitrepo.Refs, repository_id i64, mut d index.DB) []string { + if last.digest == '' || last.tips.len == 0 || refs.shallow { return []string{} } if d.scanned_locations(repository_id) or { 2 } != 1 { From 77051ed5daf0546aefa86d4956b3d49688e737a3 Mon Sep 17 00:00:00 2001 From: scher Date: Sat, 5 Sep 2026 15:35:39 +0300 Subject: [PATCH 05/11] stop a partial run from replacing a repository's membership --- index/index.v | 13 +++++++++---- index/schema.v | 12 +++++++++++- syncer/README.md | 4 +++- syncer/syncer.v | 10 ++++++---- 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/index/index.v b/index/index.v index 67a7372..5521503 100644 --- a/index/index.v +++ b/index/index.v @@ -232,8 +232,11 @@ pub fn (mut d DB) location_state(key string) !LocationState { } } +// set_location_state records what a walk left behind. It also marks the location +// walked which an invalidating call is no less evidence of: the walk that could +// not say where it stopped still put its commits in the index. pub fn (mut d DB) set_location_state(key string, digest string, tips []string) ! { - d.conn.exec_param_many('UPDATE repository_locations SET refs_digest = ?, ref_tips = ? + d.conn.exec_param_many('UPDATE repository_locations SET refs_digest = ?, ref_tips = ?, walked = 1 WHERE key = ?', [ digest, tips.join(' '), @@ -243,10 +246,12 @@ pub fn (mut d DB) set_location_state(key string, digest string, tips []string) ! // scanned_locations counts the locations of a repository that have ever been // walked. Membership is the union of them, a location can only reason about -// what a repository holds on its own when it is the only one. +// what a repository holds on its own when it is the only one. What counts is +// having walked, not holding a fingerprint: a location whose state was thrown +// away still holds the commits it added. pub fn (mut d DB) scanned_locations(repository_id i64) !int { - return d.conn.q_int("SELECT count(*) FROM repository_locations - WHERE repository_id = ${repository_id} AND refs_digest != ''")! + return d.conn.q_int('SELECT count(*) FROM repository_locations + WHERE repository_id = ${repository_id} AND walked = 1')! } // replace_remotes records every remote a repository has, as evidence. These are diff --git a/index/schema.v b/index/schema.v index d88db11..0d9d794 100644 --- a/index/schema.v +++ b/index/schema.v @@ -2,7 +2,7 @@ module index // schema_version is bumped whenever a migration is appended. A database written // by a newer gitlife is refused rather than guessed at. -const schema_version = 4 +const schema_version = 5 // migrations[v - 1] holds the statements that take the schema to version v. // Statements are listed one per entry because SQLite prepares a single statement @@ -12,6 +12,7 @@ const migrations = [ v2, v3, v4, + v5, ] const v1 = [ @@ -124,3 +125,12 @@ const v3 = []string{} const v4 = [ "ALTER TABLE repository_locations ADD COLUMN ref_tips TEXT NOT NULL DEFAULT ''", ] + +// v5 separates having been walked from holding a usable fingerprint. A location +// that was walked and later invalidated still holds the membership it added and +// a repository counting how many of its locations feed that membership has to +// count it. Existing rows carry their digest over as the answer. +const v5 = [ + 'ALTER TABLE repository_locations ADD COLUMN walked INTEGER NOT NULL DEFAULT 0', + "UPDATE repository_locations SET walked = 1 WHERE refs_digest != ''", +] diff --git a/syncer/README.md b/syncer/README.md index cbe56a9..ed41f39 100644 --- a/syncer/README.md +++ b/syncer/README.md @@ -36,7 +36,9 @@ wrote. Registration is separate from preparation because a repository can be reachable through more than one location. A working tree and a cached clone of the same origin are one repository and its membership is the union of them, not whichever -was scanned last. +was scanned last. That union is only rebuilt from scratch by a run that reaches +every location feeding it; a sync of one source adds to it instead because the +locations it never visited still hold what they reach. Work is handed out round robin and taken back in that same order, making the output of `--jobs 8` byte identical to the output of `--jobs 1`. diff --git a/syncer/syncer.v b/syncer/syncer.v index ad2f7f9..ad4cb30 100644 --- a/syncer/syncer.v +++ b/syncer/syncer.v @@ -114,7 +114,8 @@ struct Task { // when the whole history has to be walked instead. See incremental. previous []string // fresh marks the first location of its repository to be written this run. - // Only that one clears the previous membership; the rest add to it. + // Only that one clears the previous membership; the rest add to it. No + // location is fresh in a run that doesn't reach all of them. fresh bool } @@ -213,7 +214,7 @@ pub fn run(c config.Config, mut d index.DB, o Options) !Report { } tasks := register(prepare_all(p, queue), mut d, mut report, mut broken, p.now) - scan, unchanged := plan(tasks) + scan, unchanged := plan(tasks, mut d) for task in unchanged { report.add(Outcome{ source: task.source_id @@ -432,7 +433,7 @@ fn incremental(last index.LocationState, refs gitrepo.Refs, repository_id i64, m // is not walked at all; if any one of them moved, every location of it is walked // because membership is replaced from their union and a partial union would lose // commits. -fn plan(tasks []Task) ([]Task, []Task) { +fn plan(tasks []Task, mut d index.DB) ([]Task, []Task) { mut grouped := map[string][]Task{} for task in tasks { grouped[task.repository_id.str()] << task @@ -449,10 +450,11 @@ fn plan(tasks []Task) ([]Task, []Task) { // reach and a difference taken at one location would drop commits the // others still hold. alone := group.len == 1 + whole := d.scanned_locations(group[0].repository_id) or { group.len } <= group.len for i, task in group { scan << Task{ ...task - fresh: i == 0 + fresh: whole && i == 0 previous: if alone { task.previous } else { []string{} } } } From 1d02fca21d225617abb6849ff942939e8db6e871 Mon Sep 17 00:00:00 2001 From: scher Date: Sat, 5 Sep 2026 15:50:50 +0300 Subject: [PATCH 06/11] name the locations a membership was built from --- index/index.v | 8 ++++++++ syncer/syncer.v | 13 +++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/index/index.v b/index/index.v index 5521503..a56154f 100644 --- a/index/index.v +++ b/index/index.v @@ -244,6 +244,14 @@ pub fn (mut d DB) set_location_state(key string, digest string, tips []string) ! ])! } +// walked_locations names the locations of a repository that have ever been +// walked. +pub fn (mut d DB) walked_locations(repository_id i64) ![]string { + rows := d.conn.exec('SELECT key FROM repository_locations + WHERE repository_id = ${repository_id} AND walked = 1')! + return rows.map(it.val(0)) +} + // scanned_locations counts the locations of a repository that have ever been // walked. Membership is the union of them, a location can only reason about // what a repository holds on its own when it is the only one. What counts is diff --git a/syncer/syncer.v b/syncer/syncer.v index ad4cb30..c81368a 100644 --- a/syncer/syncer.v +++ b/syncer/syncer.v @@ -449,8 +449,17 @@ fn plan(tasks []Task, mut d index.DB) ([]Task, []Task) { // whole at every one of them. Its membership is the union of what they // reach and a difference taken at one location would drop commits the // others still hold. - alone := group.len == 1 - whole := d.scanned_locations(group[0].repository_id) or { group.len } <= group.len + mut here := map[string]bool{} + for task in group { + here[task.location_key] = true + } + alone := here.len == 1 + // Membership is only replaced when the run holds every location that fed + // it, named rather than counted: two sources can offer the same location + // while a third is missing and the count would call that whole. An + // unanswered question counts as a location that is missing. + fed := d.walked_locations(group[0].repository_id) or { [''] } + whole := fed.all(it in here) for i, task in group { scan << Task{ ...task From 4ab59d4d6642e942bee562598cfe42f77db4104c Mon Sep 17 00:00:00 2001 From: scher Date: Sat, 5 Sep 2026 15:51:02 +0300 Subject: [PATCH 07/11] walk the history that is there --- gitrepo/README.md | 6 +++++- gitrepo/env.v | 8 ++++++-- gitrepo/gitrepo.v | 8 ++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/gitrepo/README.md b/gitrepo/README.md index 295a173..3ac3ab1 100644 --- a/gitrepo/README.md +++ b/gitrepo/README.md @@ -19,4 +19,8 @@ fetch can deepen a shallow repository without moving a ref and what it puts in reach are ancestors of the very tips a difference would exclude. Every walk shares one ref scope. An incremental walk that disagreed with the full -one about which refs count would add commits that a later full walk removes. +one about which refs count would add commits that a later full walk removes. The +scope leaves out what is not history: the stash, notes and replacements. A graft +rewrites the graph git reports, so what is indexed is the history that is +actually there, which is also the only one two walks taken at different times +can agree on. diff --git a/gitrepo/env.v b/gitrepo/env.v index 2d50385..cbb3848 100644 --- a/gitrepo/env.v +++ b/gitrepo/env.v @@ -22,9 +22,13 @@ const scrubbed = ['GIT_TRACE', 'GIT_TRACE_CURL', 'GIT_TRACE_PACKET', 'GIT_TRACE_ // Their credential.helper is neutralized per invocation instead. pub fn git_env(extra map[string]string) map[string]string { mut add := { - 'GIT_TRACE_REDACT': '1' + 'GIT_TRACE_REDACT': '1' // An unattended sync must never block on a prompt. - 'GIT_TERMINAL_PROMPT': '0' + 'GIT_TERMINAL_PROMPT': '0' + // git substitutes replaced objects by default. What is indexed is the + // history that is actually there which is also the only one two walks + // taken at different times can agree on. + 'GIT_NO_REPLACE_OBJECTS': '1' } for name, value in extra { add[name] = value diff --git a/gitrepo/gitrepo.v b/gitrepo/gitrepo.v index 56c4ac9..edfdecc 100644 --- a/gitrepo/gitrepo.v +++ b/gitrepo/gitrepo.v @@ -13,8 +13,11 @@ const commit_fields = 11 // ref_scope is what a scan considers reachable. Every walk uses the same // scope or an incremental one would disagree with the full one about what a -// repository holds. -const ref_scope = ['--exclude=refs/stash', '--exclude=refs/notes/*', '--all'] +// repository holds. Replacements are left out with the rest of what is not +// history: a graft rewrites the graph git reports and a difference taken across +// a graft being added or dropped would be a difference between two graphs. +const ref_scope = ['--exclude=refs/stash', '--exclude=refs/notes/*', '--exclude=refs/replace/*', + '--all'] pub struct Commit { pub: @@ -122,6 +125,7 @@ pub fn (g Git) refs(dir string) !Refs { // in_scope mirrors ref_scope which git applies and we can't ask it about. fn in_scope(name string) bool { return name != 'refs/stash' && !name.starts_with('refs/notes/') + && !name.starts_with('refs/replace/') } // add_tip keeps one entry per object. Ten branches on one commit are one tip. From 82d6082b2ba6ecfe6acfc380e8e2676783c72a8d Mon Sep 17 00:00:00 2001 From: scher Date: Sat, 5 Sep 2026 16:11:02 +0300 Subject: [PATCH 08/11] walk past a graft file and distrust what was walked through one --- gitrepo/README.md | 8 ++++---- gitrepo/env.v | 8 +++++--- index/schema.v | 12 +++++++++++- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/gitrepo/README.md b/gitrepo/README.md index 3ac3ab1..edd8d9b 100644 --- a/gitrepo/README.md +++ b/gitrepo/README.md @@ -20,7 +20,7 @@ reach are ancestors of the very tips a difference would exclude. Every walk shares one ref scope. An incremental walk that disagreed with the full one about which refs count would add commits that a later full walk removes. The -scope leaves out what is not history: the stash, notes and replacements. A graft -rewrites the graph git reports, so what is indexed is the history that is -actually there, which is also the only one two walks taken at different times -can agree on. +scope leaves out what is not history: the stash, notes and replacements. A +replacement and the older graft file both rewrite the graph git reports, and both +are turned off, so what is indexed is the history that is actually there, which +is also the only one two walks taken at different times can agree on. diff --git a/gitrepo/env.v b/gitrepo/env.v index cbb3848..4bd36be 100644 --- a/gitrepo/env.v +++ b/gitrepo/env.v @@ -25,10 +25,12 @@ pub fn git_env(extra map[string]string) map[string]string { 'GIT_TRACE_REDACT': '1' // An unattended sync must never block on a prompt. 'GIT_TERMINAL_PROMPT': '0' - // git substitutes replaced objects by default. What is indexed is the - // history that is actually there which is also the only one two walks - // taken at different times can agree on. + // git rewrites ancestry from replacements and from the graft file before + // it reports any of it. What is indexed is the history that is actually + // there which is also the only one two walks taken at different times + // can agree on. 'GIT_NO_REPLACE_OBJECTS': '1' + 'GIT_GRAFT_FILE': '/dev/null' } for name, value in extra { add[name] = value diff --git a/index/schema.v b/index/schema.v index 0d9d794..74f3b3c 100644 --- a/index/schema.v +++ b/index/schema.v @@ -2,7 +2,7 @@ module index // schema_version is bumped whenever a migration is appended. A database written // by a newer gitlife is refused rather than guessed at. -const schema_version = 5 +const schema_version = 6 // migrations[v - 1] holds the statements that take the schema to version v. // Statements are listed one per entry because SQLite prepares a single statement @@ -13,6 +13,7 @@ const migrations = [ v3, v4, v5, + v6, ] const v1 = [ @@ -134,3 +135,12 @@ const v5 = [ 'ALTER TABLE repository_locations ADD COLUMN walked INTEGER NOT NULL DEFAULT 0', "UPDATE repository_locations SET walked = 1 WHERE refs_digest != ''", ] + +// v6 drops every fingerprint taken while git was still rewriting ancestry from +// replacements and grafts. Those locations were fingerprinted for a different +// question and hold whatever the rewritten graph reported. One full walk each +// puts the history that is really there in its place. What they contributed +// stands until then which is the safe direction to be wrong in. +const v6 = [ + "UPDATE repository_locations SET refs_digest = '', ref_tips = ''", +] From a7df65c42c68cd46289e9b0fef77a4ad98d34a26 Mon Sep 17 00:00:00 2001 From: scher Date: Sat, 5 Sep 2026 16:25:00 +0300 Subject: [PATCH 09/11] make a truncated history say so --- gitrepo/gitrepo.v | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gitrepo/gitrepo.v b/gitrepo/gitrepo.v index edfdecc..d3864ee 100644 --- a/gitrepo/gitrepo.v +++ b/gitrepo/gitrepo.v @@ -115,10 +115,14 @@ pub fn (g Git) refs(dir string) !Refs { oid := if answers.len > 1 { answers[1] } else { '' } add_tip(mut tips, mut seen, oid) lines << 'HEAD ' + oid + shallow := answers.len > 0 && answers[0] == 'true' + if shallow { + lines << 'shallow' + } return Refs{ digest: sha256.sum256(lines.join('\n').bytes()).hex() - tips: tips - shallow: answers.len > 0 && answers[0] == 'true' + tips: if shallow { []string{} } else { tips } + shallow: shallow } } From 5286d94589c244edcdb02a37fe397193236101da Mon Sep 17 00:00:00 2001 From: scher Date: Sat, 5 Sep 2026 16:38:53 +0300 Subject: [PATCH 10/11] record nothing for a walk that still owes a removal --- index/schema.v | 5 +++++ syncer/syncer.v | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/index/schema.v b/index/schema.v index 74f3b3c..d9b6c13 100644 --- a/index/schema.v +++ b/index/schema.v @@ -134,6 +134,11 @@ const v4 = [ const v5 = [ 'ALTER TABLE repository_locations ADD COLUMN walked INTEGER NOT NULL DEFAULT 0', "UPDATE repository_locations SET walked = 1 WHERE refs_digest != ''", + + "UPDATE repository_locations SET walked = 1 + WHERE kind = 'remote' AND repository_id IN ( + SELECT dc.repository_id FROM discoveries dc JOIN sources s ON s.id = dc.source_id + WHERE s.kind IN ('git', 'github'))", ] // v6 drops every fingerprint taken while git was still rewriting ancestry from diff --git a/syncer/syncer.v b/syncer/syncer.v index c81368a..959f460 100644 --- a/syncer/syncer.v +++ b/syncer/syncer.v @@ -117,6 +117,10 @@ struct Task { // Only that one clears the previous membership; the rest add to it. No // location is fresh in a run that doesn't reach all of them. fresh bool + // whole says the run holds every location that fed this repository's + // membership. A run that doesn't can only add to that membership, so it still + // owes it a removal and records no state for what it walked. + whole bool } // Scanned is a worker's walk of one task's commits. It lives from the moment its @@ -464,6 +468,7 @@ fn plan(tasks []Task, mut d index.DB) ([]Task, []Task) { scan << Task{ ...task fresh: whole && i == 0 + whole: whole previous: if alone { task.previous } else { []string{} } } } @@ -578,7 +583,7 @@ fn store(task Task, scan Scanned, mut d index.DB, mut report Report, mut broken return } } - if scan.refs.digest == task.refs.digest { + if task.whole && scan.refs.digest == task.refs.digest { d.set_location_state(task.location_key, task.refs.digest, task.refs.tips) or {} } else { d.set_location_state(task.location_key, '', []string{}) or {} From cfbf53004a444227f6a1f6b21b35b172cdfa4308 Mon Sep 17 00:00:00 2001 From: scher Date: Sat, 5 Sep 2026 16:51:33 +0300 Subject: [PATCH 11/11] let a location claim its share before it writes one --- index/index.v | 8 ++++++++ index/schema.v | 1 + syncer/syncer.v | 5 +++++ 3 files changed, 14 insertions(+) diff --git a/index/index.v b/index/index.v index a56154f..7b5baa3 100644 --- a/index/index.v +++ b/index/index.v @@ -244,6 +244,14 @@ pub fn (mut d DB) set_location_state(key string, digest string, tips []string) ! ])! } +// mark_walked records that a location is about to add to its repository's +// membership. It runs before the membership is written rather than after: a run +// that stopped in between would otherwise leave commits behind that no location +// admits to and a later run would replace the membership they belong to. +pub fn (mut d DB) mark_walked(key string) ! { + d.conn.exec_param('UPDATE repository_locations SET walked = 1 WHERE key = ?', key)! +} + // walked_locations names the locations of a repository that have ever been // walked. pub fn (mut d DB) walked_locations(repository_id i64) ![]string { diff --git a/index/schema.v b/index/schema.v index d9b6c13..608e32e 100644 --- a/index/schema.v +++ b/index/schema.v @@ -135,6 +135,7 @@ const v5 = [ 'ALTER TABLE repository_locations ADD COLUMN walked INTEGER NOT NULL DEFAULT 0', "UPDATE repository_locations SET walked = 1 WHERE refs_digest != ''", + "UPDATE repository_locations SET walked = 1 WHERE kind IN ('worktree', 'bare')", "UPDATE repository_locations SET walked = 1 WHERE kind = 'remote' AND repository_id IN ( SELECT dc.repository_id FROM discoveries dc JOIN sources s ON s.id = dc.source_id diff --git a/syncer/syncer.v b/syncer/syncer.v index 959f460..fc15bc5 100644 --- a/syncer/syncer.v +++ b/syncer/syncer.v @@ -563,6 +563,11 @@ fn store(task Task, scan Scanned, mut d index.DB, mut report Report, mut broken scan.error, now) return } + d.mark_walked(task.location_key) or { + fail_repository(mut d, mut report, mut broken, task.source_id, task.name, task.location_key, + err.msg(), now) + return + } mut fresh := 0 mut held := scan.commits.len if scan.delta {