walk a known repository as a difference

gitrepo/README.md

@@ -8,3 +8,19 @@ Records are read with a fixed arity of eleven lines per commit rather than a
 NUL-delimited format because V's process helpers truncate at the first NUL.
 Every calendar facing value keeps the commit's own timezone offset, which makes a
 day the day the author experienced rather than the day it was in UTC.
+
+A repository the index has seen before is walked as a difference. `refs` keeps
+the tips a walk stopped at and the next one asks git for the commits that came
+into scope since (`added`) and the ones that left it (`dropped`) rather than for
+the whole history again. Both fail when a tip is no longer in the repository,
+which is what a rewrite followed by a `git gc` leaves behind and the caller
+walks everything instead. `refs` also says whether the history is truncated: a
+fetch can deepen a shallow repository without moving a ref and what it puts in
+reach are ancestors of the very tips a difference would exclude.
+
+Every walk shares one ref scope. An incremental walk that disagreed with the full
+one about which refs count would add commits that a later full walk removes. The
+scope leaves out what is not history: the stash, notes and replacements. A
+replacement and the older graft file both rewrite the graph git reports, and both
+are turned off, so what is indexed is the history that is actually there, which
+is also the only one two walks taken at different times can agree on.

gitrepo/env.v

@@ -22,9 +22,15 @@ const scrubbed = ['GIT_TRACE', 'GIT_TRACE_CURL', 'GIT_TRACE_PACKET', 'GIT_TRACE_
 // Their credential.helper is neutralized per invocation instead.
 pub fn git_env(extra map[string]string) map[string]string {
 	mut add := {
-		'GIT_TRACE_REDACT':    '1'
+		'GIT_TRACE_REDACT':       '1'
 		// An unattended sync must never block on a prompt.
-		'GIT_TERMINAL_PROMPT': '0'
+		'GIT_TERMINAL_PROMPT':    '0'
+		// git rewrites ancestry from replacements and from the graft file before
+		// it reports any of it. What is indexed is the history that is actually
+		// there which is also the only one two walks taken at different times
+		// can agree on.
+		'GIT_NO_REPLACE_OBJECTS': '1'
+		'GIT_GRAFT_FILE':         '/dev/null'
 	}
 	for name, value in extra {
 		add[name] = value

gitrepo/gitrepo.v

@@ -11,6 +11,14 @@ const commit_format = '%H%n%P%n%an%n%ae%n%at%n%az%n%cn%n%ce%n%ct%n%cz%n%s'
 
 const commit_fields = 11
 
+// ref_scope is what a scan considers reachable. Every walk uses the same
+// scope or an incremental one would disagree with the full one about what a
+// repository holds. Replacements are left out with the rest of what is not
+// history: a graft rewrites the graph git reports and a difference taken across
+// a graft being added or dropped would be a difference between two graphs.
+const ref_scope = ['--exclude=refs/stash', '--exclude=refs/notes/*', '--exclude=refs/replace/*',
+	'--all']
+
 pub struct Commit {
 pub:
 	object_id       string
@@ -26,6 +34,18 @@ pub:
 	subject         string
 }
 
+// Refs is where a location's refs stood at a scan: the fingerprint that says
+// whether anything moved and the tips a later scan walks from.
+pub struct Refs {
+pub:
+	digest string
+	tips   []string
+	// shallow says the history is truncated. What such a repository holds can
+	// grow without a ref moving, so its digest can't be trusted to say that
+	// nothing changed.
+	shallow bool
+}
+
 pub struct Scan {
 pub:
 	object_format string
@@ -70,21 +90,55 @@ pub fn (g Git) object_format(dir string) !string {
 	return f
 }
 
-// refs_digest fingerprints every ref plus HEAD. An unchanged digest means no ref
-// moved, meaning there is nothing to walk.
-pub fn (g Git) refs_digest(dir string) !string {
+// refs fingerprints every ref plus HEAD and keeps the tips inside the scan's
+// scope. An unchanged digest means no ref moved, meaning there is nothing to
+// walk. The tips are what the next scan asks git to walk from.
+pub fn (g Git) refs(dir string) !Refs {
 	r := g.at(dir, ['for-each-ref', '--format=%(objectname) %(refname)'])!
 	mut lines := r.stdout.split_into_lines().filter(it != '')
 	lines.sort()
-	// An empty repository has no HEAD to resolve; that is not a failure.
+	mut tips := []string{}
+	mut seen := map[string]bool{}
+	for line in lines {
+		space := line.index(' ') or { continue }
+		if in_scope(line[space + 1..]) {
+			add_tip(mut tips, mut seen, line[..space])
+		}
+	}
 	head := proc.run(proc.Cmd{
 		exe: g.exe
-		args: ['rev-parse', '--verify', '--quiet', 'HEAD']
+		args: ['rev-parse', '--is-shallow-repository', '--verify', '--quiet', 'HEAD']
 		cwd: dir
 		env: git_env(map[string]string{})
 	})!
-	lines << 'HEAD ' + head.stdout.trim_space()
-	return sha256.sum256(lines.join('\n').bytes()).hex()
+	answers := head.stdout.split_into_lines().filter(it != '')
+	oid := if answers.len > 1 { answers[1] } else { '' }
+	add_tip(mut tips, mut seen, oid)
+	lines << 'HEAD ' + oid
+	shallow := answers.len > 0 && answers[0] == 'true'
+	if shallow {
+		lines << 'shallow'
+	}
+	return Refs{
+		digest:  sha256.sum256(lines.join('\n').bytes()).hex()
+		tips:    if shallow { []string{} } else { tips }
+		shallow: shallow
+	}
+}
+
+// in_scope mirrors ref_scope which git applies and we can't ask it about.
+fn in_scope(name string) bool {
+	return name != 'refs/stash' && !name.starts_with('refs/notes/')
+		&& !name.starts_with('refs/replace/')
+}
+
+// add_tip keeps one entry per object. Ten branches on one commit are one tip.
+fn add_tip(mut tips []string, mut seen map[string]bool, oid string) {
+	if oid == '' || oid in seen {
+		return
+	}
+	seen[oid] = true
+	tips << oid
 }
 
 // remotes lists a repository's remotes by name.
@@ -125,28 +179,74 @@ pub fn primary_remote(remotes map[string]string) string {
 	return ''
 }
 
-// commits walks every ref. refs/stash and refs/notes are excluded: they are
-// scratch space and metadata, not a record of work.
-//
-// The log is read as it arrives. A repository with a million commits would
-// otherwise exist three times over: git's output as one string, that string cut
-// into lines, and the commits themselves.
+// commits walks everything in scope. The log is read as it arrives: a repository
+// with a million commits would otherwise exist three times over, as git's output
+// in one string, as that string cut into lines and as the commits themselves.
 pub fn (g Git) commits(dir string) ![]Commit {
+	return g.log(dir, [])!
+}
+
+// added walks the commits that came into scope since a scan whose tips these
+// were. Everything the old tips reach is indexed already, so git is asked to
+// mark it uninteresting and print the rest.
+//
+// It fails when a tip is no longer in the repository.
+pub fn (g Git) added(dir string, previous []string) ![]Commit {
+	mut exclude := ['--not']
+	exclude << previous
+	return g.log(dir, exclude)!
+}
+
+// dropped names the commits the old tips reached that nothing reaches now: a
+// deleted branch or history rewritten out from under one. Only the object ids,
+// since these are commits the index already holds and is about to stop counting
+// for this repository.
+pub fn (g Git) dropped(dir string, previous []string) ![]string {
+	mut args := ['rev-list']
+	args << previous
+	args << '--not'
+	args << ref_scope
+	mut oids := Oids{}
+	proc.check_stream(proc.Cmd{
+		exe:  g.exe
+		args: args
+		cwd:  dir
+		env:  git_env(map[string]string{})
+	}, mut oids)!
+	return oids.list
+}
+
+fn (g Git) log(dir string, extra []string) ![]Commit {
+	mut args := ['log']
+	args << ref_scope
+	args << ['--no-use-mailmap', '--no-show-signature', '--format=' + commit_format]
+	args << extra
 	mut reader := CommitReader{}
 	proc.check_stream(proc.Cmd{
-		exe: g.exe
-		args: ['log', '--exclude=refs/stash', '--exclude=refs/notes/*', '--all', '--no-use-mailmap',
-			'--no-show-signature', '--format=' + commit_format]
-		cwd: dir
-		env: git_env(map[string]string{})
+		exe:  g.exe
+		args: args
+		cwd:  dir
+		env:  git_env(map[string]string{})
 	}, mut reader)!
 	return reader.done()
 }
 
+// An Oids collects a walk that prints one object id per line.
+struct Oids {
+mut:
+	list []string
+}
+
+fn (mut o Oids) take(line string) ! {
+	if line != '' {
+		o.list << line
+	}
+}
+
 pub fn (g Git) scan(dir string) !Scan {
 	return Scan{
 		object_format: g.object_format(dir)!
-		refs_digest: g.refs_digest(dir)!
+		refs_digest: g.refs(dir)!.digest
 		commits: g.commits(dir)!
 	}
 }

index/README.md

@@ -26,3 +26,9 @@ two reports from disagreeing about what belongs to the user.
 `purge` forgets what no configured source can still reach. It is deliberately a
 separate act from removing a source. `--dry-run` is the same code path with the
 transaction rolled back instead of committed.
+
+`write_delta` is the other way in: the commits a location gained and the ones it
+lost for a repository whose membership can only have come from that one
+location. A repository reached through several locations holds the union of them
+and one location losing sight of a commit says nothing about the others, so those
+are written as whole snapshots.