diff --git a/gitrepo/README.md b/gitrepo/README.md index e97131f..edd8d9b 100644 --- a/gitrepo/README.md +++ b/gitrepo/README.md @@ -8,3 +8,19 @@ Records are read with a fixed arity of eleven lines per commit rather than a NUL-delimited format because V's process helpers truncate at the first NUL. Every calendar facing value keeps the commit's own timezone offset, which makes a day the day the author experienced rather than the day it was in UTC. + +A repository the index has seen before is walked as a difference. `refs` keeps +the tips a walk stopped at and the next one asks git for the commits that came +into scope since (`added`) and the ones that left it (`dropped`) rather than for +the whole history again. Both fail when a tip is no longer in the repository, +which is what a rewrite followed by a `git gc` leaves behind and the caller +walks everything instead. `refs` also says whether the history is truncated: a +fetch can deepen a shallow repository without moving a ref and what it puts in +reach are ancestors of the very tips a difference would exclude. + +Every walk shares one ref scope. An incremental walk that disagreed with the full +one about which refs count would add commits that a later full walk removes. The +scope leaves out what is not history: the stash, notes and replacements. A +replacement and the older graft file both rewrite the graph git reports, and both +are turned off, so what is indexed is the history that is actually there, which +is also the only one two walks taken at different times can agree on. diff --git a/gitrepo/env.v b/gitrepo/env.v index 2d50385..4bd36be 100644 --- a/gitrepo/env.v +++ b/gitrepo/env.v @@ -22,9 +22,15 @@ const scrubbed = ['GIT_TRACE', 'GIT_TRACE_CURL', 'GIT_TRACE_PACKET', 'GIT_TRACE_ // Their credential.helper is neutralized per invocation instead. pub fn git_env(extra map[string]string) map[string]string { mut add := { - 'GIT_TRACE_REDACT': '1' + 'GIT_TRACE_REDACT': '1' // An unattended sync must never block on a prompt. - 'GIT_TERMINAL_PROMPT': '0' + 'GIT_TERMINAL_PROMPT': '0' + // git rewrites ancestry from replacements and from the graft file before + // it reports any of it. What is indexed is the history that is actually + // there which is also the only one two walks taken at different times + // can agree on. + 'GIT_NO_REPLACE_OBJECTS': '1' + 'GIT_GRAFT_FILE': '/dev/null' } for name, value in extra { add[name] = value diff --git a/gitrepo/gitrepo.v b/gitrepo/gitrepo.v index 92f6748..d3864ee 100644 --- a/gitrepo/gitrepo.v +++ b/gitrepo/gitrepo.v @@ -11,6 +11,14 @@ const commit_format = '%H%n%P%n%an%n%ae%n%at%n%az%n%cn%n%ce%n%ct%n%cz%n%s' const commit_fields = 11 +// ref_scope is what a scan considers reachable. Every walk uses the same +// scope or an incremental one would disagree with the full one about what a +// repository holds. Replacements are left out with the rest of what is not +// history: a graft rewrites the graph git reports and a difference taken across +// a graft being added or dropped would be a difference between two graphs. +const ref_scope = ['--exclude=refs/stash', '--exclude=refs/notes/*', '--exclude=refs/replace/*', + '--all'] + pub struct Commit { pub: object_id string @@ -26,6 +34,18 @@ pub: subject string } +// Refs is where a location's refs stood at a scan: the fingerprint that says +// whether anything moved and the tips a later scan walks from. +pub struct Refs { +pub: + digest string + tips []string + // shallow says the history is truncated. What such a repository holds can + // grow without a ref moving, so its digest can't be trusted to say that + // nothing changed. + shallow bool +} + pub struct Scan { pub: object_format string @@ -70,21 +90,55 @@ pub fn (g Git) object_format(dir string) !string { return f } -// refs_digest fingerprints every ref plus HEAD. An unchanged digest means no ref -// moved, meaning there is nothing to walk. -pub fn (g Git) refs_digest(dir string) !string { +// refs fingerprints every ref plus HEAD and keeps the tips inside the scan's +// scope. An unchanged digest means no ref moved, meaning there is nothing to +// walk. The tips are what the next scan asks git to walk from. +pub fn (g Git) refs(dir string) !Refs { r := g.at(dir, ['for-each-ref', '--format=%(objectname) %(refname)'])! mut lines := r.stdout.split_into_lines().filter(it != '') lines.sort() - // An empty repository has no HEAD to resolve; that is not a failure. + mut tips := []string{} + mut seen := map[string]bool{} + for line in lines { + space := line.index(' ') or { continue } + if in_scope(line[space + 1..]) { + add_tip(mut tips, mut seen, line[..space]) + } + } head := proc.run(proc.Cmd{ exe: g.exe - args: ['rev-parse', '--verify', '--quiet', 'HEAD'] + args: ['rev-parse', '--is-shallow-repository', '--verify', '--quiet', 'HEAD'] cwd: dir env: git_env(map[string]string{}) })! - lines << 'HEAD ' + head.stdout.trim_space() - return sha256.sum256(lines.join('\n').bytes()).hex() + answers := head.stdout.split_into_lines().filter(it != '') + oid := if answers.len > 1 { answers[1] } else { '' } + add_tip(mut tips, mut seen, oid) + lines << 'HEAD ' + oid + shallow := answers.len > 0 && answers[0] == 'true' + if shallow { + lines << 'shallow' + } + return Refs{ + digest: sha256.sum256(lines.join('\n').bytes()).hex() + tips: if shallow { []string{} } else { tips } + shallow: shallow + } +} + +// in_scope mirrors ref_scope which git applies and we can't ask it about. +fn in_scope(name string) bool { + return name != 'refs/stash' && !name.starts_with('refs/notes/') + && !name.starts_with('refs/replace/') +} + +// add_tip keeps one entry per object. Ten branches on one commit are one tip. +fn add_tip(mut tips []string, mut seen map[string]bool, oid string) { + if oid == '' || oid in seen { + return + } + seen[oid] = true + tips << oid } // remotes lists a repository's remotes by name. @@ -125,28 +179,74 @@ pub fn primary_remote(remotes map[string]string) string { return '' } -// commits walks every ref. refs/stash and refs/notes are excluded: they are -// scratch space and metadata, not a record of work. -// -// The log is read as it arrives. A repository with a million commits would -// otherwise exist three times over: git's output as one string, that string cut -// into lines, and the commits themselves. +// commits walks everything in scope. The log is read as it arrives: a repository +// with a million commits would otherwise exist three times over, as git's output +// in one string, as that string cut into lines and as the commits themselves. pub fn (g Git) commits(dir string) ![]Commit { + return g.log(dir, [])! +} + +// added walks the commits that came into scope since a scan whose tips these +// were. Everything the old tips reach is indexed already, so git is asked to +// mark it uninteresting and print the rest. +// +// It fails when a tip is no longer in the repository. +pub fn (g Git) added(dir string, previous []string) ![]Commit { + mut exclude := ['--not'] + exclude << previous + return g.log(dir, exclude)! +} + +// dropped names the commits the old tips reached that nothing reaches now: a +// deleted branch or history rewritten out from under one. Only the object ids, +// since these are commits the index already holds and is about to stop counting +// for this repository. +pub fn (g Git) dropped(dir string, previous []string) ![]string { + mut args := ['rev-list'] + args << previous + args << '--not' + args << ref_scope + mut oids := Oids{} + proc.check_stream(proc.Cmd{ + exe: g.exe + args: args + cwd: dir + env: git_env(map[string]string{}) + }, mut oids)! + return oids.list +} + +fn (g Git) log(dir string, extra []string) ![]Commit { + mut args := ['log'] + args << ref_scope + args << ['--no-use-mailmap', '--no-show-signature', '--format=' + commit_format] + args << extra mut reader := CommitReader{} proc.check_stream(proc.Cmd{ - exe: g.exe - args: ['log', '--exclude=refs/stash', '--exclude=refs/notes/*', '--all', '--no-use-mailmap', - '--no-show-signature', '--format=' + commit_format] - cwd: dir - env: git_env(map[string]string{}) + exe: g.exe + args: args + cwd: dir + env: git_env(map[string]string{}) }, mut reader)! return reader.done() } +// An Oids collects a walk that prints one object id per line. +struct Oids { +mut: + list []string +} + +fn (mut o Oids) take(line string) ! { + if line != '' { + o.list << line + } +} + pub fn (g Git) scan(dir string) !Scan { return Scan{ object_format: g.object_format(dir)! - refs_digest: g.refs_digest(dir)! + refs_digest: g.refs(dir)!.digest commits: g.commits(dir)! } } diff --git a/index/README.md b/index/README.md index 98efbe3..a74c34d 100644 --- a/index/README.md +++ b/index/README.md @@ -26,3 +26,9 @@ two reports from disagreeing about what belongs to the user. `purge` forgets what no configured source can still reach. It is deliberately a separate act from removing a source. `--dry-run` is the same code path with the transaction rolled back instead of committed. + +`write_delta` is the other way in: the commits a location gained and the ones it +lost for a repository whose membership can only have come from that one +location. A repository reached through several locations holds the union of them +and one location losing sight of a commit says nothing about the others, so those +are written as whole snapshots. diff --git a/index/index.v b/index/index.v index 651aaad..7b5baa3 100644 --- a/index/index.v +++ b/index/index.v @@ -212,23 +212,64 @@ fn (mut d DB) repository_of_location(id i64) !i64 { return rows[0].val(0).i64() } -// location_digest is the fingerprint of the refs seen the last time this location -// was scanned successfully. -pub fn (mut d DB) location_digest(key string) !string { - rows := d.conn.exec_param('SELECT refs_digest FROM repository_locations WHERE key = ?', key)! +// LocationState is what the last successful scan of a location left behind: the +// fingerprint of the refs it saw and the tips it stopped at. +pub struct LocationState { +pub: + digest string + tips []string +} + +pub fn (mut d DB) location_state(key string) !LocationState { + rows := d.conn.exec_param('SELECT refs_digest, ref_tips FROM repository_locations WHERE key = ?', + key)! if rows.len == 0 { - return '' + return LocationState{} + } + return LocationState{ + digest: rows[0].val(0) + tips: rows[0].val(1).split(' ').filter(it != '') } - return rows[0].val(0) } -pub fn (mut d DB) set_location_digest(key string, digest string) ! { - d.conn.exec_param_many('UPDATE repository_locations SET refs_digest = ? WHERE key = ?', [ +// set_location_state records what a walk left behind. It also marks the location +// walked which an invalidating call is no less evidence of: the walk that could +// not say where it stopped still put its commits in the index. +pub fn (mut d DB) set_location_state(key string, digest string, tips []string) ! { + d.conn.exec_param_many('UPDATE repository_locations SET refs_digest = ?, ref_tips = ?, walked = 1 + WHERE key = ?', [ digest, + tips.join(' '), key, ])! } +// mark_walked records that a location is about to add to its repository's +// membership. It runs before the membership is written rather than after: a run +// that stopped in between would otherwise leave commits behind that no location +// admits to and a later run would replace the membership they belong to. +pub fn (mut d DB) mark_walked(key string) ! { + d.conn.exec_param('UPDATE repository_locations SET walked = 1 WHERE key = ?', key)! +} + +// walked_locations names the locations of a repository that have ever been +// walked. +pub fn (mut d DB) walked_locations(repository_id i64) ![]string { + rows := d.conn.exec('SELECT key FROM repository_locations + WHERE repository_id = ${repository_id} AND walked = 1')! + return rows.map(it.val(0)) +} + +// scanned_locations counts the locations of a repository that have ever been +// walked. Membership is the union of them, a location can only reason about +// what a repository holds on its own when it is the only one. What counts is +// having walked, not holding a fingerprint: a location whose state was thrown +// away still holds the commits it added. +pub fn (mut d DB) scanned_locations(repository_id i64) !int { + return d.conn.q_int('SELECT count(*) FROM repository_locations + WHERE repository_id = ${repository_id} AND walked = 1')! +} + // replace_remotes records every remote a repository has, as evidence. These are // deliberately not locations: a non primary remote must never merge two // repositories or adding 'upstream' to a fork would fuse it with what it forked. @@ -327,18 +368,99 @@ pub fn (mut d DB) write_snapshot(repository_id i64, scan gitrepo.Scan, fresh boo fn (mut d DB) apply_snapshot(repository_id i64, scan gitrepo.Scan, fresh bool) !int { before := d.conn.q_int('SELECT COALESCE(MAX(id), 0) FROM commits')! + d.insert_commits(scan.object_format, scan.commits)! + + // Membership is diffed against what the location holds, not replaced by it. + d.conn.exec('DELETE FROM scanned_commits')! + mut scanned := Batch{ + query: 'INSERT OR IGNORE INTO scanned_commits (commit_id) + SELECT id FROM commits WHERE object_format = ? AND object_id = ?' + } + for c in scan.commits { + scanned.add(mut d, [scan.object_format, c.object_id])! + } + scanned.send(mut d)! + + id := repository_id.str() + d.conn.exec_param('INSERT OR IGNORE INTO repository_commits (repository_id, commit_id) + SELECT ?, commit_id FROM scanned_commits', id)! + if fresh { + d.conn.exec_param('DELETE FROM repository_commits + WHERE repository_id = ? + AND commit_id NOT IN (SELECT commit_id FROM scanned_commits)', id)! + } + + d.conn.exec_param_many('UPDATE repositories SET object_format = ? WHERE id = ?', [ + scan.object_format, + id, + ])! + return int(d.conn.q_int('SELECT COALESCE(MAX(id), 0) FROM commits')! - before) +} + +// write_delta records what changed for a location instead of what it holds: +// the commits that came into scope and the ones that left it. It returns how +// many commits the index had never seen and how many the repository holds now. +// +// Only a repository with a single scanned location can be written this way. A +// commit leaving one location's scope says nothing about whether another +// location still reaches it, and membership is the union of them. +pub fn (mut d DB) write_delta(repository_id i64, object_format string, added []gitrepo.Commit, dropped []string) !(int, int) { + d.conn.begin()! + fresh, held := d.apply_delta(repository_id, object_format, added, dropped) or { + d.conn.rollback() or {} + return err + } + d.conn.commit()! + return fresh, held +} + +fn (mut d DB) apply_delta(repository_id i64, object_format string, added []gitrepo.Commit, dropped []string) !(int, int) { + before := d.conn.q_int('SELECT COALESCE(MAX(id), 0) FROM commits')! + d.insert_commits(object_format, added)! + + id := repository_id.str() + mut members := Batch{ + query: 'INSERT OR IGNORE INTO repository_commits (repository_id, commit_id) + VALUES (?, (SELECT id FROM commits WHERE object_format = ? AND object_id = ?))' + } + for c in added { + members.add(mut d, [id, object_format, c.object_id])! + } + members.send(mut d)! + + // A commit nothing reaches any more stops counting for this repository. The + // commit row itself stays: another repository may hold it and purge is what + // forgets a commit for good. + mut gone := Batch{ + query: 'DELETE FROM repository_commits + WHERE repository_id = ? + AND commit_id = (SELECT id FROM commits WHERE object_format = ? AND object_id = ?)' + } + for oid in dropped { + gone.add(mut d, [id, object_format, oid])! + } + gone.send(mut d)! + + held := d.conn.q_int('SELECT count(*) FROM repository_commits WHERE repository_id = ${repository_id}')! + return int(d.conn.q_int('SELECT COALESCE(MAX(id), 0) FROM commits')! - before), held +} + +// insert_commits writes the commits themselves and the identities they name. +// A commit row is never updated, so the same commit reached twice costs a lookup +// and nothing else. +fn (mut d DB) insert_commits(object_format string, commits []gitrepo.Commit) ! { mut idents := Batch{ query: 'INSERT OR IGNORE INTO git_identities (name, email, email_norm) VALUES (?, ?, ?)' } mut seen := map[string]bool{} - for c in scan.commits { + for c in commits { add_identity(mut idents, mut d, mut seen, c.author_name, c.author_email)! add_identity(mut idents, mut d, mut seen, c.committer_name, c.committer_email)! } idents.send(mut d)! - mut commits := Batch{ + mut rows := Batch{ query: "INSERT OR IGNORE INTO commits ( object_format, object_id, parents, author_identity_id, author_time, author_tz, author_date, @@ -349,9 +471,9 @@ fn (mut d DB) apply_snapshot(repository_id i64, scan gitrepo.Scan, fresh bool) ! (SELECT id FROM git_identities WHERE name = ? AND email = ?), ?, ?, date(?, 'unixepoch'), ?)" } - for c in scan.commits { - commits.add(mut d, [ - scan.object_format, + for c in commits { + rows.add(mut d, [ + object_format, c.object_id, c.parents, c.author_name, @@ -367,33 +489,7 @@ fn (mut d DB) apply_snapshot(repository_id i64, scan gitrepo.Scan, fresh bool) ! c.subject, ])! } - commits.send(mut d)! - - // Membership is diffed against what the location holds, not replaced by it. - d.conn.exec('DELETE FROM scanned_commits')! - mut scanned := Batch{ - query: 'INSERT OR IGNORE INTO scanned_commits (commit_id) - SELECT id FROM commits WHERE object_format = ? AND object_id = ?' - } - for c in scan.commits { - scanned.add(mut d, [scan.object_format, c.object_id])! - } - scanned.send(mut d)! - - id := repository_id.str() - d.conn.exec_param('INSERT OR IGNORE INTO repository_commits (repository_id, commit_id) - SELECT ?, commit_id FROM scanned_commits', id)! - if fresh { - d.conn.exec_param('DELETE FROM repository_commits - WHERE repository_id = ? - AND commit_id NOT IN (SELECT commit_id FROM scanned_commits)', id)! - } - - d.conn.exec_param_many('UPDATE repositories SET object_format = ? WHERE id = ?', [ - scan.object_format, - id, - ])! - return int(d.conn.q_int('SELECT COALESCE(MAX(id), 0) FROM commits')! - before) + rows.send(mut d)! } // rows_per_batch is how many rows an insert holds at once. The statement is diff --git a/index/schema.v b/index/schema.v index ecb40d9..608e32e 100644 --- a/index/schema.v +++ b/index/schema.v @@ -2,7 +2,7 @@ module index // schema_version is bumped whenever a migration is appended. A database written // by a newer gitlife is refused rather than guessed at. -const schema_version = 3 +const schema_version = 6 // migrations[v - 1] holds the statements that take the schema to version v. // Statements are listed one per entry because SQLite prepares a single statement @@ -11,6 +11,9 @@ const migrations = [ v1, v2, v3, + v4, + v5, + v6, ] const v1 = [ @@ -115,3 +118,35 @@ const v2 = [ // can belong to two repository rows that have to be merged first. That work is // fold_transport_keys in index.v. const v3 = []string{} + +// v4 keeps the ref tips a location was last scanned at, next to the digest that +// says whether they moved. A scan that knows where the last one stopped can ask +// git for the difference instead of the whole history. Existing rows start +// empty, costing each location one full walk and nothing after that. +const v4 = [ + "ALTER TABLE repository_locations ADD COLUMN ref_tips TEXT NOT NULL DEFAULT ''", +] + +// v5 separates having been walked from holding a usable fingerprint. A location +// that was walked and later invalidated still holds the membership it added and +// a repository counting how many of its locations feed that membership has to +// count it. Existing rows carry their digest over as the answer. +const v5 = [ + 'ALTER TABLE repository_locations ADD COLUMN walked INTEGER NOT NULL DEFAULT 0', + "UPDATE repository_locations SET walked = 1 WHERE refs_digest != ''", + + "UPDATE repository_locations SET walked = 1 WHERE kind IN ('worktree', 'bare')", + "UPDATE repository_locations SET walked = 1 + WHERE kind = 'remote' AND repository_id IN ( + SELECT dc.repository_id FROM discoveries dc JOIN sources s ON s.id = dc.source_id + WHERE s.kind IN ('git', 'github'))", +] + +// v6 drops every fingerprint taken while git was still rewriting ancestry from +// replacements and grafts. Those locations were fingerprinted for a different +// question and hold whatever the rewritten graph reported. One full walk each +// puts the history that is really there in its place. What they contributed +// stands until then which is the safe direction to be wrong in. +const v6 = [ + "UPDATE repository_locations SET refs_digest = '', ref_tips = ''", +] diff --git a/syncer/README.md b/syncer/README.md index 492bb6d..ed41f39 100644 --- a/syncer/README.md +++ b/syncer/README.md @@ -11,7 +11,7 @@ database, and no worker thread ever holds the connection. discover what repositories exist (serial, one call per source) prepare clone or fetch, fingerprint refs (parallel, no database) register resolve identity, decide what moved (serial, database) -read walk the commits (parallel, no database) +read walk the commits, or what changed (parallel, no database) write bring the snapshots up to date (serial, database) ``` @@ -21,10 +21,24 @@ held from the moment its worker finished with it until it is in the database, and not a moment longer, so a run costs a history per worker rather than one per repository. +A location the index has walked before is walked as a difference: its stored ref +tips say where the last walk stopped and git is asked for what came into scope +since and what left it. A location seen for the first time, one whose old tips +have been garbage collected, any repository with more than one walked location +and any repository whose history is truncated are walked whole. + +A walk records where it stopped only when the refs it was handed are still the +refs it ended on. A repository that moved while it was being read was walked at +a state nobody fingerprinted, its location is marked unwalked and the next +sync walks it whole rather than trusting a fingerprint of a history it never +wrote. + Registration is separate from preparation because a repository can be reachable through more than one location. A working tree and a cached clone of the same origin are one repository and its membership is the union of them, not whichever -was scanned last. +was scanned last. That union is only rebuilt from scratch by a run that reaches +every location feeding it; a sync of one source adds to it instead because the +locations it never visited still hold what they reach. Work is handed out round robin and taken back in that same order, making the output of `--jobs 8` byte identical to the output of `--jobs 1`. diff --git a/syncer/syncer.v b/syncer/syncer.v index 4d69feb..fc15bc5 100644 --- a/syncer/syncer.v +++ b/syncer/syncer.v @@ -93,7 +93,7 @@ struct Prepared { action string remotes map[string]string object_format string - digest string + refs gitrepo.Refs elapsed_ms int error string } @@ -106,19 +106,35 @@ struct Task { location_key string dir string object_format string - digest string + refs gitrepo.Refs action string elapsed_ms int // what preparing it already cost changed bool + // previous holds the tips this location was last walked at and is empty + // when the whole history has to be walked instead. See incremental. + previous []string // fresh marks the first location of its repository to be written this run. - // Only that one clears the previous membership; the rest add to it. + // Only that one clears the previous membership; the rest add to it. No + // location is fresh in a run that doesn't reach all of them. fresh bool + // whole says the run holds every location that fed this repository's + // membership. A run that doesn't can only add to that membership, so it still + // owes it a removal and records no state for what it walked. + whole bool } // Scanned is a worker's walk of one task's commits. It lives from the moment its // worker hands it over until the writer is done with it and no longer. struct Scanned { - commits []gitrepo.Commit + commits []gitrepo.Commit + // dropped names commits that left the location's scope and delta says + // whether commits is everything the location holds or only what is new. + dropped []string + delta bool + // refs is the ref state read again once the walk was done. The walk is only + // allowed to say where the next one may start from when the refs it ended on + // are still the refs it was handed. + refs gitrepo.Refs elapsed_ms int error string } @@ -202,7 +218,7 @@ pub fn run(c config.Config, mut d index.DB, o Options) !Report { } tasks := register(prepare_all(p, queue), mut d, mut report, mut broken, p.now) - scan, unchanged := plan(tasks) + scan, unchanged := plan(tasks, mut d) for task in unchanged { report.add(Outcome{ source: task.source_id @@ -321,7 +337,7 @@ fn gather(p Pass, job Job) !Prepared { action: action remotes: remotes object_format: p.git.object_format(dir)! - digest: p.git.refs_digest(dir)! + refs: p.git.refs(dir)! } } @@ -377,6 +393,7 @@ fn enter(item Prepared, mut d index.DB, now i64) !Task { } scanned := locations[0].key + last := d.location_state(scanned)! return Task{ source_id: item.source_id repository_id: repository_id @@ -384,19 +401,43 @@ fn enter(item Prepared, mut d index.DB, now i64) !Task { location_key: scanned dir: item.dir object_format: item.object_format - digest: item.digest + refs: item.refs action: item.action elapsed_ms: item.elapsed_ms - changed: item.digest != d.location_digest(scanned)! + // A shallow repository is never called unchanged. A fetch can deepen it + // without moving a ref, leaving the digest saying nothing happened + // while the history it holds has grown. + changed: item.refs.digest != last.digest || item.refs.shallow + previous: incremental(last, item.refs, repository_id, mut d) } } +// incremental decides whether this location can be walked as a difference and +// answers with the tips to walk from or with nothing. +// +// Three conditions. The location must have been walked before or there is no +// difference to take. It must be the only location of its repository the index +// has ever walked: membership is the union of a repository's locations and +// locations that join that union in this run are ruled out in plan which is the +// first place the whole run is known. And the history must be whole: deepening a +// shallow repository puts ancestors of the old tips in reach and a difference +// taken from those tips calls exactly those ancestors uninteresting. +fn incremental(last index.LocationState, refs gitrepo.Refs, repository_id i64, mut d index.DB) []string { + if last.digest == '' || last.tips.len == 0 || refs.shallow { + return []string{} + } + if d.scanned_locations(repository_id) or { 2 } != 1 { + return []string{} + } + return last.tips +} + // plan splits the registered tasks into the ones to walk and the ones to leave // alone. A repository whose locations all still hold the refs they held last time // is not walked at all; if any one of them moved, every location of it is walked // because membership is replaced from their union and a partial union would lose // commits. -fn plan(tasks []Task) ([]Task, []Task) { +fn plan(tasks []Task, mut d index.DB) ([]Task, []Task) { mut grouped := map[string][]Task{} for task in tasks { grouped[task.repository_id.str()] << task @@ -408,19 +449,48 @@ fn plan(tasks []Task) ([]Task, []Task) { unchanged << group continue } + // A repository reached through more than one location this run is walked + // whole at every one of them. Its membership is the union of what they + // reach and a difference taken at one location would drop commits the + // others still hold. + mut here := map[string]bool{} + for task in group { + here[task.location_key] = true + } + alone := here.len == 1 + // Membership is only replaced when the run holds every location that fed + // it, named rather than counted: two sources can offer the same location + // while a third is missing and the count would call that whole. An + // unanswered question counts as a location that is missing. + fed := d.walked_locations(group[0].repository_id) or { [''] } + whole := fed.all(it in here) for i, task in group { scan << Task{ ...task - fresh: i == 0 + fresh: whole && i == 0 + whole: whole + previous: if alone { task.previous } else { []string{} } } } } return scan, unchanged } -// read walks one location's commits and times the walk. +// read walks one location and times the walk. A location the index has seen +// before is walked as a difference: what came into scope since the last walk and +// what left it. Everything else is walked whole. fn read(p Pass, task Task) Scanned { started := time.ticks() + if task.previous.len > 0 { + if delta := read_delta(p, task) { + return Scanned{ + ...delta + refs: p.git.refs(task.dir) or { gitrepo.Refs{} } + elapsed_ms: int(time.ticks() - started) + } + } + // The old tips are gone + } commits := p.git.commits(task.dir) or { return Scanned{ error: err.msg() @@ -428,10 +498,20 @@ fn read(p Pass, task Task) Scanned { } return Scanned{ commits: commits + refs: p.git.refs(task.dir) or { gitrepo.Refs{} } elapsed_ms: int(time.ticks() - started) } } +fn read_delta(p Pass, task Task) !Scanned { + dropped := p.git.dropped(task.dir, task.previous)! + return Scanned{ + commits: p.git.added(task.dir, task.previous)! + dropped: dropped + delta: true + } +} + // read_stride is one worker's share, handed over one walk at a time. Same round // robin as prepare_stride, for the same reason. fn read_stride(p Pass, tasks []Task, first int, stride int, out chan Scanned) { @@ -483,22 +563,42 @@ fn store(task Task, scan Scanned, mut d index.DB, mut report Report, mut broken scan.error, now) return } - fresh := d.write_snapshot(task.repository_id, gitrepo.Scan{ - object_format: task.object_format - refs_digest: task.digest - commits: scan.commits - }, task.fresh) or { + d.mark_walked(task.location_key) or { fail_repository(mut d, mut report, mut broken, task.source_id, task.name, task.location_key, err.msg(), now) return } - d.set_location_digest(task.location_key, task.digest) or {} + mut fresh := 0 + mut held := scan.commits.len + if scan.delta { + fresh, held = d.write_delta(task.repository_id, task.object_format, scan.commits, + scan.dropped) or { + fail_repository(mut d, mut report, mut broken, task.source_id, task.name, + task.location_key, err.msg(), now) + return + } + } else { + fresh = d.write_snapshot(task.repository_id, gitrepo.Scan{ + object_format: task.object_format + refs_digest: task.refs.digest + commits: scan.commits + }, task.fresh) or { + fail_repository(mut d, mut report, mut broken, task.source_id, task.name, + task.location_key, err.msg(), now) + return + } + } + if task.whole && scan.refs.digest == task.refs.digest { + d.set_location_state(task.location_key, task.refs.digest, task.refs.tips) or {} + } else { + d.set_location_state(task.location_key, '', []string{}) or {} + } report.add(Outcome{ source: task.source_id repository: task.name status: 'ok' action: task.action - commits: scan.commits.len + commits: held new_commits: fresh elapsed_ms: task.elapsed_ms + scan.elapsed_ms })