From 05321c85d4e94e5cf8d24d2d9aed6408012ef9fc Mon Sep 17 00:00:00 2001 From: Lucca Oliveira Date: Fri, 3 Jul 2026 08:58:04 -0300 Subject: [PATCH 01/12] test(slickGitLog): cover commit tuple mapping, grafts and helpers Promote the nested remove_trailing_space and the graft-line split (as parseGraftLine) to gitLogToDB methods, behavior unchanged, so they are reachable from tests. The suite exercises them directly plus git_commits_iterator/findGrafts/isBare against JGit fixture repos with pinned identities and dates. --- slickGitLog/build.sbt | 3 +- slickGitLog/src/main/scala/gitLogToDb.scala | 14 +- .../src/test/scala/gitLogToDbSpec.scala | 160 ++++++++++++++++++ 3 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 slickGitLog/src/test/scala/gitLogToDbSpec.scala diff --git a/slickGitLog/build.sbt b/slickGitLog/build.sbt index b1cacaf3..24c0fa56 100644 --- a/slickGitLog/build.sbt +++ b/slickGitLog/build.sbt @@ -7,7 +7,8 @@ libraryDependencies ++= Seq( "com.typesafe.slick" %% "slick" % "3.0.0", "org.xerial" % "sqlite-jdbc" % "3.45.3.0", "com.zaxxer" % "HikariCP" % "2.4.1", - "org.eclipse.jgit" % "org.eclipse.jgit" % "4.6.0.201612231935-r" + "org.eclipse.jgit" % "org.eclipse.jgit" % "4.6.0.201612231935-r", + "org.scalatest" %% "scalatest" % "3.0.8" % "test" ) resolvers ++= Seq( diff --git a/slickGitLog/src/main/scala/gitLogToDb.scala b/slickGitLog/src/main/scala/gitLogToDb.scala index 5d6ad566..29f808f9 100644 --- a/slickGitLog/src/main/scala/gitLogToDb.scala +++ b/slickGitLog/src/main/scala/gitLogToDb.scala @@ -139,6 +139,13 @@ object gitLogToDB extends ProgramInfo { def commitsPerOp = 10000 + def remove_trailing_space(st:String) = st.replaceAll(" $", "") + + def parseGraftLine(l: String): (String, Int, String) = { + val f = l.split(' ') + (f(1), 1, f(0)) + } + def git_commits_iterator(git:Git) = { val logs = git.log.all.call() @@ -165,8 +172,6 @@ object gitLogToDB extends ProgramInfo { val aut = l.getAuthorIdent() val com = l.getCommitterIdent() - def remove_trailing_space(st:String) = st.replaceAll(" $", "") - ( // first is the commit tuple // second is the log tuple @@ -205,10 +210,7 @@ object gitLogToDB extends ProgramInfo { // we'll see if ((new File(graftsFileName)).exists) { - Source.fromFile(graftsFileName).getLines.map { l => - val f = l.split(' ') - (f(1), 1, f(0)) - }.toList + Source.fromFile(graftsFileName).getLines.map(parseGraftLine).toList } else { List() } diff --git a/slickGitLog/src/test/scala/gitLogToDbSpec.scala b/slickGitLog/src/test/scala/gitLogToDbSpec.scala new file mode 100644 index 00000000..a7a6849e --- /dev/null +++ b/slickGitLog/src/test/scala/gitLogToDbSpec.scala @@ -0,0 +1,160 @@ +/* + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +import org.scalatest.FunSuite + +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.lib.PersonIdent + +import java.io.File +import java.io.PrintWriter +import java.nio.file.Files +import java.text.SimpleDateFormat +import java.util.{Date, TimeZone} + +class gitLogToDbSpec extends FunSuite { + + test("remove_trailing_space strips exactly one trailing space") { + assert(gitLogToDB.remove_trailing_space("Bob ") === "Bob") + assert(gitLogToDB.remove_trailing_space("Bob") === "Bob") + // regex is " $": only the last space is removed + assert(gitLogToDB.remove_trailing_space("Bob ") === "Bob ") + // inner spaces are kept + assert(gitLogToDB.remove_trailing_space("Bob Smith") === "Bob Smith") + } + + test("parseGraftLine maps ' ' to (parent, 1, child)") { + val child = "c" * 40 + val parent = "p" * 40 + assert(gitLogToDB.parseGraftLine(s"$child $parent") === (parent, 1, child)) + } + + // --- fixture helpers ----------------------------------------------------- + + def withTempRepo(testCode: (Git, File) => Unit): Unit = { + val dir = Files.createTempDirectory("gitLogToDbSpec").toFile + val git = Git.init.setDirectory(dir).call() + try { + testCode(git, dir) + } finally { + git.close() + def rm(f: File): Unit = { + if (f.isDirectory) f.listFiles.foreach(rm) + f.delete() + } + rm(dir) + } + } + + def commitFile(git: Git, dir: File, name: String, content: String, + who: PersonIdent, message: String) = { + val out = new PrintWriter(new File(dir, name)) + out.print(content) + out.close() + git.add.addFilepattern(name).call() + git.commit.setAuthor(who).setCommitter(who).setMessage(message).call() + } + + val utc = TimeZone.getTimeZone("UTC") + val aliceDate = new Date(1500000000000L) + val bobDate = new Date(1500000600000L) + // author name with a trailing space, to exercise remove_trailing_space + val alice = new PersonIdent("Alice Coder ", "alice@example.com", aliceDate, utc) + val bob = new PersonIdent("Bob Hacker", "bob@example.com", bobDate, utc) + + // ------------------------------------------------------------------------- + + test("git_commits_iterator maps commits to the expected tuples") { + withTempRepo { (git, dir) => + val c1 = commitFile(git, dir, "a.txt", "one\n", alice, "first commit") + val c2 = commitFile(git, dir, "b.txt", "two\n", bob, + "second commit\n\nSigned-off-by: Bob Hacker \n") + + val windows = gitLogToDB.git_commits_iterator(git).toList + assert(windows.size === 1) // 2 commits fit in one window of commitsPerOp + + val commits = windows(0).sortBy(_._1._4) // order by author date + assert(commits.size === 2) + + val (commit1, log1, parents1, footers1) = commits(0) + val (commit2, log2, parents2, footers2) = commits(1) + + // expected dates computed with the same formatter over the same Date, + // so the assertion is immune to the JVM default timezone + val dt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + + assert(commit1 === (c1.getName, + "Alice Coder", "alice@example.com", dt.format(aliceDate), + "Alice Coder", "alice@example.com", dt.format(aliceDate), + "first commit", false)) + assert(log1 === (c1.getName, "first commit")) + assert(parents1 === Seq()) + assert(footers1 === Seq()) + + assert(commit2._1 === c2.getName) + assert(commit2._8 === "second commit") + assert(commit2._9 === false) + assert(parents2 === Seq((c2.getName, 0, c1.getName))) + assert(footers2 === Seq((c2.getName, 0, "Signed-off-by", + "Bob Hacker "))) + } + } + + test("a merge commit has ismerge true and two indexed parents") { + withTempRepo { (git, dir) => + val base = commitFile(git, dir, "a.txt", "one\n", alice, "base") + git.checkout.setCreateBranch(true).setName("side").call() + val side = commitFile(git, dir, "b.txt", "two\n", bob, "side") + git.checkout.setName("master").call() + val main = commitFile(git, dir, "c.txt", "three\n", alice, "main") + val merge = git.merge.include(side).call().getNewHead + + val all = gitLogToDB.git_commits_iterator(git).toList.flatten + val mergeTuple = all.find(_._1._1 == merge.getName).get + + assert(mergeTuple._1._9 === true) // ismerge + assert(mergeTuple._3 === Seq( + (merge.getName, 0, main.getName), + (merge.getName, 1, side.getName))) + } + } + + test("findGrafts parses info/grafts in a non-bare repo") { + withTempRepo { (git, dir) => + val child = "1" * 40 + val parent = "2" * 40 + new File(dir, ".git/info").mkdirs() + val out = new PrintWriter(new File(dir, ".git/info/grafts")) + out.println(s"$child $parent") + out.close() + + assert(gitLogToDB.findGrafts(dir.getPath, git) === List((parent, 1, child))) + } + } + + test("findGrafts returns the empty list when no grafts file exists") { + withTempRepo { (git, dir) => + assert(gitLogToDB.findGrafts(dir.getPath, git) === List()) + } + } + + test("isBare is false for a working-tree repo") { + withTempRepo { (git, dir) => + assert(gitLogToDB.isBare(git) === false) + } + } +} From 75c6972d174814a1863080107e6b32a2aa7c24ae Mon Sep 17 00:00:00 2001 From: Lucca Oliveira Date: Fri, 3 Jul 2026 08:58:04 -0300 Subject: [PATCH 02/12] test(persons): cover identity normalization and unification Lift the nested splitEmail/dealWithSingleWords defs and extract the transitive email-merge fold (unifyByEmail) and preferred-name pick (preferredName) out of main, behavior unchanged. The suite pins down strip_accents, Person equality semantics, key building, and the transitive merge that is the heart of the module. --- persons/build.sbt | 3 +- persons/src/main/scala/unifyPersons.scala | 66 +++++---- persons/src/test/scala/unifyPersonsSpec.scala | 139 ++++++++++++++++++ 3 files changed, 180 insertions(+), 28 deletions(-) create mode 100644 persons/src/test/scala/unifyPersonsSpec.scala diff --git a/persons/build.sbt b/persons/build.sbt index c410b9a2..1a8b78a4 100644 --- a/persons/build.sbt +++ b/persons/build.sbt @@ -7,7 +7,8 @@ libraryDependencies ++= Seq( "org.xerial" % "sqlite-jdbc" % "3.45.3.0", "com.zaxxer" % "HikariCP" % "2.4.1", "org.eclipse.jgit" % "org.eclipse.jgit" % "4.6.0.201612231935-r", - "info.folone" %% "poi-scala" % "0.18" + "info.folone" %% "poi-scala" % "0.18", + "org.scalatest" %% "scalatest" % "3.0.8" % "test" ) resolvers ++= Seq( diff --git a/persons/src/main/scala/unifyPersons.scala b/persons/src/main/scala/unifyPersons.scala index dbc79a2d..6c23a196 100644 --- a/persons/src/main/scala/unifyPersons.scala +++ b/persons/src/main/scala/unifyPersons.scala @@ -212,6 +212,43 @@ object unifyPersons { } } + def splitEmail(st:String) = { + val fields = st.split('@') + if (fields.size > 1) { + (fields(0), fields(1)) + } else { + (fields(0), "") + } + + } + + def dealWithSingleWords(key:String, addon: String)= { + // we don't like names that don't have spaces + // since they are usually reused (eg. Jim, root, etc) + // so instead, use the other field + val noacc = strip_accents(key) + if (noacc.contains(' ')) + noacc.toLowerCase + else (noacc+" at " +addon).toLowerCase + } + + // unify by common email: merge groups of persons that share at least one + // lowercased email, transitively + def unifyByEmail(setsNames: Iterable[Iterable[Person]]): Set[Set[Person]] = { + setsNames.foldLeft(Set.empty[Set[Person]])((cum, curi) => { + val cur = curi.toSet + val curEmails = cur.map{_.lcEmail} + val (hasCommon, rest) = cum.partition(_.map{_.lcEmail} & curEmails nonEmpty) + rest + (cur ++ hasCommon.flatten) + }) + } + + // we prefer names that contain a space (single words are usually + // reused, e.g. Jim, root); otherwise fall back to the email + def preferredName(v: List[Person]): String = { + if (v(0).name.contains(" ")) v(0).name else v(0).email + } + // return an iterator that returns, for each commit // a tuple of the author and the committer info def git_commits_iterator(git:Git) = { @@ -220,16 +257,6 @@ object unifyPersons { val logsIt = logs.asScala.toIterator - def splitEmail(st:String) = { - val fields = st.split('@') - if (fields.size > 1) { - (fields(0), fields(1)) - } else { - (fields(0), "") - } - - } - logsIt.map { l => val author = l.getAuthorIdent().getEmailAddress val committer = l.getCommitterIdent().getEmailAddress @@ -239,16 +266,6 @@ object unifyPersons { val authorName = l.getAuthorIdent().getName val committerName = l.getCommitterIdent().getName - def dealWithSingleWords(key:String, addon: String)= { - // we don't like names that don't have spaces - // since they are usually reused (eg. Jim, root, etc) - // so instead, use the other field - val noacc = strip_accents(key) - if (noacc.contains(' ')) - noacc.toLowerCase - else (noacc+" at " +addon).toLowerCase - } - val authorKey = dealWithSingleWords(authorName, author) val commKey = dealWithSingleWords(committerName, committer) @@ -426,12 +443,7 @@ object unifyPersons { println("Unifying by email...") // unify by common email - val unifiedByEmail = setsNames.foldLeft(Set.empty[Set[Person]])((cum, curi) => { - val cur = curi.toSet - val curEmails = cur.map{_.lcEmail} - val (hasCommon, rest) = cum.partition(_.map{_.lcEmail} & curEmails nonEmpty) - rest + (cur ++ hasCommon.flatten) - }) + val unifiedByEmail = unifyByEmail(setsNames) println(s" ... reduced to ${unifiedByEmail.size} emails") @@ -455,7 +467,7 @@ object unifyPersons { // attach the count of all, authored, committed val keys = mapByKey.map{ case (k,v) => - val nameToUse = if (v(0).name.contains(" ")) v(0).name else v(0).email + val nameToUse = preferredName(v) val identCount = v.size val countAll = v.map{ e => (everybody(e), diff --git a/persons/src/test/scala/unifyPersonsSpec.scala b/persons/src/test/scala/unifyPersonsSpec.scala new file mode 100644 index 00000000..636252b1 --- /dev/null +++ b/persons/src/test/scala/unifyPersonsSpec.scala @@ -0,0 +1,139 @@ +/* + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +import org.scalatest.FunSuite +import unifyPersons.Person + +class unifyPersonsSpec extends FunSuite { + + def mkPerson(name: String, email: String): Person = { + val (user, domain) = unifyPersons.splitEmail(email) + val key = unifyPersons.dealWithSingleWords(name, email) + new Person(name, key, email, email.toLowerCase, user, domain) + } + + // strip_accents + + test("strip_accents removes combining diacritical marks") { + assert(unifyPersons.strip_accents("José") === "Jose") + assert(unifyPersons.strip_accents("Éléonore Müller") === "Eleonore Muller") + } + + test("strip_accents maps the special characters ø ß æ ð") { + assert(unifyPersons.strip_accents("Løvberg") === "Lovberg") + assert(unifyPersons.strip_accents("Straße") === "Strasse") + assert(unifyPersons.strip_accents("æ") === "ae") + assert(unifyPersons.strip_accents("ð") === "o") + } + + test("strip_accents leaves plain ascii untouched") { + assert(unifyPersons.strip_accents("John Smith") === "John Smith") + } + + // Person equality + + test("Persons with identical fields are equal and hash-equal") { + val a = mkPerson("Jim Smith", "jim@example.com") + val b = mkPerson("Jim Smith", "jim@example.com") + assert(a === b) + assert(a.hashCode === b.hashCode) + } + + test("Persons differing in email are not equal") { + val a = mkPerson("Jim Smith", "jim@example.com") + val b = mkPerson("Jim Smith", "jim@other.org") + assert(a !== b) + } + + test("equal Persons deduplicate in a Set") { + val a = mkPerson("Jim Smith", "jim@example.com") + val b = mkPerson("Jim Smith", "jim@example.com") + val c = mkPerson("Ann Lee", "ann@example.com") + assert(Set(a, b, c).size === 2) + } + + // splitEmail + + test("splitEmail splits user and domain") { + assert(unifyPersons.splitEmail("a@b.com") === ("a", "b.com")) + } + + test("splitEmail without @ yields empty domain") { + assert(unifyPersons.splitEmail("localonly") === ("localonly", "")) + } + + // dealWithSingleWords + + test("a name with a space becomes the lowercased name") { + assert(unifyPersons.dealWithSingleWords("Jim Smith", "jim@x.com") === "jim smith") + } + + test("a single-word name falls back to 'name at addon'") { + assert(unifyPersons.dealWithSingleWords("root", "root@x.com") === "root at root@x.com") + } + + test("accents are stripped before building the key") { + assert(unifyPersons.dealWithSingleWords("José Núñez", "jose@x.com") === "jose nunez") + } + + // unifyByEmail + + test("unifyByEmail merges groups sharing an email, transitively") { + val jim1 = mkPerson("Jim Smith", "jim@example.com") + val jim2 = mkPerson("James Smith", "jim@example.com") // shares email with jim1 + val jim3 = mkPerson("James Smith", "jsmith@work.org") // shares name-group with jim2 + val ann = mkPerson("Ann Lee", "ann@example.org") + + // groups as produced by the group-by-name step + val groups = List(List(jim1), List(jim2, jim3), List(ann)) + + val unified = unifyPersons.unifyByEmail(groups) + + assert(unified.size === 2) + assert(unified.contains(Set(jim1, jim2, jim3))) + assert(unified.contains(Set(ann))) + } + + test("unifyByEmail keeps disjoint groups apart") { + val a = mkPerson("Jim Smith", "jim@example.com") + val b = mkPerson("Ann Lee", "ann@example.org") + val unified = unifyPersons.unifyByEmail(List(List(a), List(b))) + assert(unified === Set(Set(a), Set(b))) + } + + test("unifyByEmail on empty input yields the empty set") { + assert(unifyPersons.unifyByEmail(Nil) === Set.empty[Set[Person]]) + } + + // preferredName + + test("preferredName prefers a name containing a space") { + val p = mkPerson("Jim Smith", "jim@example.com") + assert(unifyPersons.preferredName(List(p)) === "Jim Smith") + } + + test("preferredName falls back to the email for single-word names") { + val p = mkPerson("root", "root@example.com") + assert(unifyPersons.preferredName(List(p)) === "root@example.com") + } + + test("preferredName only considers the first (most used) identity") { + val single = mkPerson("root", "root@example.com") + val full = mkPerson("Rudy Root", "rudy@example.com") + assert(unifyPersons.preferredName(List(single, full)) === "root@example.com") + } +} From 1cf827cea502047b753198a09f9bec336f6bda54 Mon Sep 17 00:00:00 2001 From: Lucca Oliveira Date: Fri, 3 Jul 2026 08:58:04 -0300 Subject: [PATCH 03/12] test(remapCommits): cover Former-commit-id footer extraction Extract the footer parsing from git_commits_iterator into extractOriginalCid(cid, message), behavior unchanged, and pin down its edge cases: footer on/not on the last line, empty and newline-only messages, uppercase or short shas falling back to cid. --- remapCommits/build.sbt | 3 +- .../src/main/scala/remapCommits.scala | 39 +++++------ .../src/test/scala/remapCommitsSpec.scala | 64 +++++++++++++++++++ 3 files changed, 87 insertions(+), 19 deletions(-) create mode 100644 remapCommits/src/test/scala/remapCommitsSpec.scala diff --git a/remapCommits/build.sbt b/remapCommits/build.sbt index b1cacaf3..24c0fa56 100644 --- a/remapCommits/build.sbt +++ b/remapCommits/build.sbt @@ -7,7 +7,8 @@ libraryDependencies ++= Seq( "com.typesafe.slick" %% "slick" % "3.0.0", "org.xerial" % "sqlite-jdbc" % "3.45.3.0", "com.zaxxer" % "HikariCP" % "2.4.1", - "org.eclipse.jgit" % "org.eclipse.jgit" % "4.6.0.201612231935-r" + "org.eclipse.jgit" % "org.eclipse.jgit" % "4.6.0.201612231935-r", + "org.scalatest" %% "scalatest" % "3.0.8" % "test" ) resolvers ++= Seq( diff --git a/remapCommits/src/main/scala/remapCommits.scala b/remapCommits/src/main/scala/remapCommits.scala index 8ebf4a71..fa883c0f 100644 --- a/remapCommits/src/main/scala/remapCommits.scala +++ b/remapCommits/src/main/scala/remapCommits.scala @@ -69,33 +69,36 @@ object remapCommits extends ProgramInfo { def commitsPerOp = 10000 + def extractOriginalCid(cid: String, message: String): String = { + + val lastline = + // split returns empty list if the string contains + // only separators. Weird. + try { + message.split("\n").last + } catch { + case _: Exception => "" + } + + val exp = "Former-commit-id: ([0-9a-f]{40})".r + + lastline match { + case exp(fcid) => fcid + case _ => cid + } + } + def git_commits_iterator(git:Git) = { val logs = git.log.all.call() val logsIt = logs.asScala.toIterator - + val mapped = logsIt.map { l => val cid = l.getName - val message = l.getFullMessage() - - val lastline = - // split returns empty list if the string contains - // only separators. Weird. - try { - message.split("\n").last - } catch { - case _: Exception => "" - } - - val exp = "Former-commit-id: ([0-9a-f]{40})".r - - val originalcid = lastline match { - case exp(fcid) => fcid - case _ => cid - } + val originalcid = extractOriginalCid(cid, l.getFullMessage()) (// must return 3 elements, because that is what the database expects cid, originalcid, null diff --git a/remapCommits/src/test/scala/remapCommitsSpec.scala b/remapCommits/src/test/scala/remapCommitsSpec.scala new file mode 100644 index 00000000..0c0e3f30 --- /dev/null +++ b/remapCommits/src/test/scala/remapCommitsSpec.scala @@ -0,0 +1,64 @@ +/* + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +import org.scalatest.FunSuite + +class remapCommitsSpec extends FunSuite { + + val cid = "a" * 40 + val originalCid = "0123456789abcdef0123456789abcdef01234567" + + test("footer on the last line yields the original commit id") { + val message = s"some subject\n\nFormer-commit-id: $originalCid" + assert(remapCommits.extractOriginalCid(cid, message) === originalCid) + } + + test("footer as the only line yields the original commit id") { + val message = s"Former-commit-id: $originalCid" + assert(remapCommits.extractOriginalCid(cid, message) === originalCid) + } + + test("footer not on the last line falls back to cid") { + val message = s"subject\nFormer-commit-id: $originalCid\ntrailing line" + assert(remapCommits.extractOriginalCid(cid, message) === cid) + } + + test("message without a footer falls back to cid") { + assert(remapCommits.extractOriginalCid(cid, "just a subject") === cid) + } + + test("empty message falls back to cid") { + assert(remapCommits.extractOriginalCid(cid, "") === cid) + } + + test("message of only newlines falls back to cid") { + // String.split("\n") on "\n" returns an empty array; .last throws and + // the catch turns it into "" -> no match -> cid + assert(remapCommits.extractOriginalCid(cid, "\n") === cid) + assert(remapCommits.extractOriginalCid(cid, "\n\n\n") === cid) + } + + test("uppercase hex footer is not recognized (regex is lowercase-only)") { + val upper = originalCid.toUpperCase + assert(remapCommits.extractOriginalCid(cid, s"Former-commit-id: $upper") === cid) + } + + test("short sha in footer is not recognized") { + val message = "Former-commit-id: 0123456789abcdef" + assert(remapCommits.extractOriginalCid(cid, message) === cid) + } +} From 311a38c791d1dcbdcd351650b387e1612c0cd5a1 Mon Sep 17 00:00:00 2001 From: Lucca Oliveira Date: Fri, 3 Jul 2026 08:58:21 -0300 Subject: [PATCH 04/12] fix(tokenize): repair the unusable unknown-parser check The autodetect path guarded on "defined defined($parsers{language})" - a doubled defined() over a bareword key - which is always true, so the check could never fire. Use the intended $parsers{$language} lookup. Defensive only: today every extension in %extensions maps to a defined parser, so no observable behavior changes. --- tokenize/tokenize.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokenize/tokenize.pl b/tokenize/tokenize.pl index d421f2d9..9b65044b 100755 --- a/tokenize/tokenize.pl +++ b/tokenize/tokenize.pl @@ -99,7 +99,7 @@ my $ext = lc($1); $language = $extensions{$ext}; Usage("Unknown extension [$ext] in file [$filename]. You must provide language using --language option") unless defined $language and $language ne ""; - Usage("Unknown parser for extension [$ext] in file [$filename]. You must provide language using --language option") unless defined defined($parsers{language}); + Usage("Unknown parser for extension [$ext] in file [$filename]. You must provide language using --language option") unless defined($parsers{$language}); } else { # check the extension exists if (not (defined $parsers{$language}) or ($parsers{$language} eq "")) { From 8e1876a7921ebd362b304d9b9b2b0f8720d7a902 Mon Sep 17 00:00:00 2001 From: Lucca Oliveira Date: Fri, 3 Jul 2026 08:58:21 -0300 Subject: [PATCH 05/12] test(perl): add black-box Test::More suites for the pipeline scripts prove-run t/ suites for tokenizeSrcMl.pl (golden files mirroring the srcMLtoken pattern, committed under tokenize/t/expected/), tokenize.pl (dispatch parity with the direct tokenizer run), tokenBySha.pl (stubbed BFG_TOKENIZE_CMD: memoization, cache hits, extension mapping, env validation) and formatBlame.pl/blameRepoFiles.pl (on-the-fly fixture repos with pinned dates; blame format, renames, filtering, overwrite). All tests run the scripts as subprocesses; no script refactoring. prettyPrint is deliberately not covered (needs fixture SQLite DBs for low marginal value). --- blameRepo/t/blameRepoFiles.t | 76 + blameRepo/t/formatBlame.t | 104 ++ tokenize/t/expected/StringUtil.java.token | 605 ++++++ tokenize/t/expected/main.c.nopos.token | 2070 +++++++++++++++++++++ tokenize/t/expected/main.c.token | 2070 +++++++++++++++++++++ tokenize/t/tokenize.t | 67 + tokenize/t/tokenizeSrcMl.t | 95 + tokenizeByBlobId/t/tokenBySha.t | 138 ++ 8 files changed, 5225 insertions(+) create mode 100644 blameRepo/t/blameRepoFiles.t create mode 100644 blameRepo/t/formatBlame.t create mode 100644 tokenize/t/expected/StringUtil.java.token create mode 100644 tokenize/t/expected/main.c.nopos.token create mode 100644 tokenize/t/expected/main.c.token create mode 100644 tokenize/t/tokenize.t create mode 100644 tokenize/t/tokenizeSrcMl.t create mode 100644 tokenizeByBlobId/t/tokenBySha.t diff --git a/blameRepo/t/blameRepoFiles.t b/blameRepo/t/blameRepoFiles.t new file mode 100644 index 00000000..1d4097af --- /dev/null +++ b/blameRepo/t/blameRepoFiles.t @@ -0,0 +1,76 @@ +#!/usr/bin/env perl + +# Tests for blameRepoFiles.pl: walks `git ls-files`, filters by regexp and +# runs formatBlame.pl per file, skipping files whose .blame already exists. + +use strict; +use warnings; +use Test::More tests => 9; +use FindBin; +use File::Temp qw(tempdir); + +my $script = "$FindBin::Bin/../blameRepoFiles.pl"; +my $workdir = tempdir(CLEANUP => 1); + +# deterministic, hermetic git +$ENV{GIT_CONFIG_NOSYSTEM} = 1; +$ENV{GIT_CONFIG_GLOBAL} = '/dev/null'; +$ENV{GIT_AUTHOR_DATE} = '2020-01-01T00:00:00 +0000'; +$ENV{GIT_COMMITTER_DATE} = '2020-01-01T00:00:00 +0000'; +$ENV{GIT_AUTHOR_NAME} = 'Alice'; +$ENV{GIT_AUTHOR_EMAIL} = 'alice@example.com'; +$ENV{GIT_COMMITTER_NAME} = 'Alice'; +$ENV{GIT_COMMITTER_EMAIL} = 'alice@example.com'; + +sub git { + my ($repo, @args) = @_; + my $cmd = "git -C '$repo' " . join(' ', @args); + system($cmd) == 0 or die "git failed: $cmd"; +} + +sub write_file { + my ($path, $content) = @_; + open(my $fh, '>', $path) or die $!; + print $fh $content; + close $fh; +} + +# fixture: two .c files and one .txt file in a single commit +my $repo = "$workdir/repo"; +mkdir $repo or die $!; +git($repo, "init -q -b main"); +write_file("$repo/a.c", "int a;\n"); +write_file("$repo/b.c", "int b;\n"); +write_file("$repo/notes.txt", "hello\n"); +git($repo, "add ."); +git($repo, "commit -q -m first"); + +my $out = "$workdir/blame-out"; +mkdir $out or die $!; + +# first pass: both .c files processed, .txt filtered out +{ + my $stdout = `perl '$script' '$repo' '$out' '\\.c\$' 2>'$workdir/stderr'`; + is($?, 0, "blameRepoFiles.pl succeeds"); + ok(-f "$out/a.c.blame", "a.c.blame created"); + ok(-f "$out/b.c.blame", "b.c.blame created"); + ok(!-e "$out/notes.txt.blame", "notes.txt is filtered out by the regexp"); + like($stdout, qr/Newly processed \[2\] Already done \[0\] files Error \[0\]/, + "summary reports two newly processed files"); +} + +# second pass without --overwrite: everything is already done +{ + my $stdout = `perl '$script' '$repo' '$out' '\\.c\$' 2>/dev/null`; + is($?, 0, "second run succeeds"); + like($stdout, qr/Newly processed \[0\] Already done \[2\] files Error \[0\]/, + "existing .blame files are skipped"); +} + +# --overwrite reprocesses everything +{ + my $stdout = `perl '$script' --overwrite '$repo' '$out' '\\.c\$' 2>/dev/null`; + is($?, 0, "overwrite run succeeds"); + like($stdout, qr/Newly processed \[2\] Already done \[0\] files Error \[0\]/, + "--overwrite reprocesses the files"); +} diff --git a/blameRepo/t/formatBlame.t b/blameRepo/t/formatBlame.t new file mode 100644 index 00000000..1a1c36e0 --- /dev/null +++ b/blameRepo/t/formatBlame.t @@ -0,0 +1,104 @@ +#!/usr/bin/env perl + +# Tests for formatBlame.pl: builds a small git repo on the fly (deterministic +# via GIT_AUTHOR_*/GIT_COMMITTER_* env vars) and checks the .blame output +# format: one line per source line, ";;\t". + +use strict; +use warnings; +use Test::More tests => 12; +use FindBin; +use File::Temp qw(tempdir); + +my $script = "$FindBin::Bin/../formatBlame.pl"; +my $workdir = tempdir(CLEANUP => 1); + +# deterministic, hermetic git +$ENV{GIT_CONFIG_NOSYSTEM} = 1; +$ENV{GIT_CONFIG_GLOBAL} = '/dev/null'; +$ENV{GIT_AUTHOR_DATE} = '2020-01-01T00:00:00 +0000'; +$ENV{GIT_COMMITTER_DATE} = '2020-01-01T00:00:00 +0000'; +$ENV{GIT_COMMITTER_NAME} = 'Committer'; +$ENV{GIT_COMMITTER_EMAIL} = 'committer@example.com'; + +sub git { + my ($repo, @args) = @_; + my $cmd = "git -C '$repo' " . join(' ', @args); + my $out = `$cmd`; + die "git failed: $cmd" if $? != 0; + chomp $out; + return $out; +} + +sub commit_as { + my ($repo, $name, $message) = @_; + local $ENV{GIT_AUTHOR_NAME} = $name; + local $ENV{GIT_AUTHOR_EMAIL} = lc($name) . '@example.com'; + git($repo, "commit -q -m '$message'"); + return git($repo, "rev-parse HEAD"); +} + +sub write_file { + my ($path, $content) = @_; + open(my $fh, '>', $path) or die $!; + print $fh $content; + close $fh; +} + +sub slurp_lines { + my ($file) = @_; + open(my $fh, '<', $file) or die "unable to read [$file]: $!"; + my @lines = <$fh>; + chomp @lines; + return @lines; +} + +# fixture: commit 1 (Alice) writes two lines, commit 2 (Bob) appends a third +my $repo = "$workdir/repo"; +mkdir $repo or die $!; +git($repo, "init -q -b main"); +write_file("$repo/f.c", "int one;\nint two;\n"); +git($repo, "add f.c"); +my $cid1 = commit_as($repo, "Alice", "first"); +write_file("$repo/f.c", "int one;\nint two;\nint three;\n"); +git($repo, "add f.c"); +my $cid2 = commit_as($repo, "Bob", "second"); + +# basic blame formatting +{ + my $dest = tempdir(CLEANUP => 1); + my $status = system("perl '$script' '$repo' f.c '$dest' 2>'$workdir/stderr'"); + is($status, 0, "formatBlame.pl succeeds"); + ok(-f "$dest/f.c.blame", "creates /f.c.blame"); + + my @lines = slurp_lines("$dest/f.c.blame"); + is(scalar(@lines), 3, "one blame line per source line"); + is($lines[0], "$cid1;;\tint one;", "line 1 blamed on the first commit"); + is($lines[1], "$cid1;;\tint two;", "line 2 blamed on the first commit"); + is($lines[2], "$cid2;;\tint three;", "line 3 blamed on the second commit"); +} + +# a custom --blameExtension is honored +{ + my $dest = tempdir(CLEANUP => 1); + my $status = system("perl '$script' --blameExtension=.tok '$repo' f.c '$dest' 2>/dev/null"); + is($status, 0, "formatBlame.pl with --blameExtension succeeds"); + ok(-f "$dest/f.c.tok", "creates /f.c.tok"); +} + +# after a rename, lines blamed on pre-rename commits carry the old filename +{ + git($repo, "mv f.c g.c"); + my $cid3 = commit_as($repo, "Alice", "rename"); + + my $dest = tempdir(CLEANUP => 1); + my $status = system("perl '$script' '$repo' g.c '$dest' 2>/dev/null"); + is($status, 0, "formatBlame.pl on the renamed file succeeds"); + + my @lines = slurp_lines("$dest/g.c.blame"); + is(scalar(@lines), 3, "renamed file still has three blame lines"); + is($lines[0], "$cid1;f.c;\tint one;", + "pre-rename lines carry the original filename"); + is($lines[2], "$cid2;f.c;\tint three;", + "all pre-rename commits report the old name"); +} diff --git a/tokenize/t/expected/StringUtil.java.token b/tokenize/t/expected/StringUtil.java.token new file mode 100644 index 00000000..22e622dd --- /dev/null +++ b/tokenize/t/expected/StringUtil.java.token @@ -0,0 +1,605 @@ +-:-|begin_unit|revision:1.0.0;language:Java;cregit-version:0.0.1 +-:-|begin_package +1:-|DECL|package|utility +1:1|package|package +1:9|name|utility +1:16|package|; +-:-|end_package +-:-| +-:-|begin_import +3:1|import|import +3:8|name|gnu +3:11|operator|. +3:12|name|trove +3:17|operator|. +3:18|name|TIntArrayList +3:31|import|; +-:-|end_import +-:-| +-:-|begin_class +5:-|DECL|class|StringUtil +5:1|specifier|public +5:8|class|class +5:14|name|StringUtil +5:25|block|{ +6:-|DECL|field|NewLineString +6:9|specifier|public +6:16|specifier|static +6:23|specifier|final +6:29|name|String +6:36|name|NewLineString +6:50|init|= +6:52|name|System +6:58|operator|. +6:59|name|getProperty +6:70|argument_list|( +6:71|literal|"line.separator" +6:87|argument_list|) +6:88|decl_stmt|; +6:90|comment|//$NON-NLS-1$ +7:-|DECL|method|findAll (int[] poss, String str, int target) +7:9|specifier|public +7:16|specifier|static +7:23|name|int +7:27|name|findAll +7:34|parameter_list|( +7:35|name|int +7:38|index|[] +7:41|name|poss +7:45|parameter_list|, +7:47|name|String +7:54|name|str +7:57|parameter_list|, +7:59|name|int +7:63|name|target +7:69|parameter_list|) +7:71|block|{ +8:17|if|if +8:20|condition|( +8:21|name|poss +8:25|operator|. +8:26|name|length +8:33|operator|== +8:36|literal|0 +8:37|condition|) +8:39|block|{ +9:25|return|return +9:32|literal|0 +9:33|return|; +10:17|block|} +12:17|name|int +12:21|name|i +12:23|init|= +12:25|literal|0 +12:26|decl_stmt|; +13:17|name|int +13:21|name|count +13:27|init|= +13:29|literal|0 +13:30|decl_stmt|; +14:17|while|while +14:23|condition|( +14:24|name|i +14:26|operator|< +14:28|name|str +14:31|operator|. +14:32|name|length +14:38|argument_list|() +14:40|condition|) +14:42|block|{ +15:25|name|int +15:29|name|p +15:31|init|= +15:33|name|str +15:36|operator|. +15:37|name|indexOf +15:44|argument_list|( +15:45|name|target +15:51|argument_list|, +15:53|name|i +15:54|argument_list|) +15:55|decl_stmt|; +16:25|if|if +16:28|condition|( +16:29|name|p +16:31|operator|< +16:33|literal|0 +16:34|condition|) +16:36|block|{ +17:33|for|for +17:37|control|( +17:38|name|int +17:42|name|c +17:44|init|= +17:46|name|count +17:51|init|; +17:53|name|c +17:55|operator|< +17:57|name|poss +17:61|operator|. +17:62|name|length +17:68|condition|; +17:70|operator|++ +17:72|name|c +17:73|control|) +17:75|block|{ +18:41|name|poss +18:45|index|[ +18:46|name|c +18:47|index|] +18:49|operator|= +18:51|operator|- +18:52|literal|1 +18:53|expr_stmt|; +19:33|block|} +20:33|return|return +20:40|name|count +20:45|return|; +21:25|block|} +22:25|name|poss +22:29|index|[ +22:30|name|count +22:35|index|] +22:37|operator|= +22:39|name|p +22:40|expr_stmt|; +23:25|operator|++ +23:27|name|count +23:32|expr_stmt|; +24:25|name|i +24:27|operator|= +24:29|name|p +24:31|operator|+ +24:33|literal|1 +24:34|expr_stmt|; +25:17|block|} +26:17|return|return +26:24|name|count +26:29|return|; +27:9|block|} +28:-|DECL|method|split (String str, int sepChar) +28:9|specifier|public +28:16|specifier|static +28:23|name|String +28:29|index|[] +28:32|name|split +28:37|parameter_list|( +28:38|name|String +28:45|name|str +28:48|parameter_list|, +28:50|name|int +28:54|name|sepChar +28:61|parameter_list|) +28:63|block|{ +29:17|name|TIntArrayList +29:31|name|sepPoss +29:39|init|= +29:41|operator|new +29:45|name|TIntArrayList +29:58|argument_list|() +29:60|decl_stmt|; +30:17|name|int +30:21|name|pos +30:25|init|= +30:27|literal|0 +30:28|decl_stmt|; +31:17|while|while +31:23|condition|( +31:24|name|pos +31:28|operator|< +31:30|name|str +31:33|operator|. +31:34|name|length +31:40|argument_list|() +31:42|condition|) +31:44|block|{ +32:25|name|int +32:29|name|q +32:31|init|= +32:33|name|str +32:36|operator|. +32:37|name|indexOf +32:44|argument_list|( +32:45|name|sepChar +32:52|argument_list|, +32:54|name|pos +32:57|argument_list|) +32:58|decl_stmt|; +33:25|if|if +33:28|condition|( +33:29|name|q +33:31|operator|!= +33:34|operator|- +33:35|literal|1 +33:36|condition|) +33:38|block|{ +34:33|name|sepPoss +34:40|operator|. +34:41|name|add +34:44|argument_list|( +34:45|name|q +34:46|argument_list|) +34:47|expr_stmt|; +35:33|name|pos +35:37|operator|= +35:39|name|q +35:41|operator|+ +35:43|literal|1 +35:44|expr_stmt|; +36:25|block|} +37:25|else|else +37:30|block|{ +38:33|name|sepPoss +38:40|operator|. +38:41|name|add +38:44|argument_list|( +38:45|name|str +38:48|operator|. +38:49|name|length +38:55|argument_list|() +38:57|argument_list|) +38:58|expr_stmt|; +39:33|name|pos +39:37|operator|= +39:39|name|str +39:42|operator|. +39:43|name|length +39:49|argument_list|() +39:51|expr_stmt|; +40:25|block|} +41:17|block|} +42:17|name|int +42:20|index|[] +42:23|name|poss +42:28|init|= +42:30|name|sepPoss +42:37|operator|. +42:38|name|toNativeArray +42:51|argument_list|() +42:53|decl_stmt|; +43:17|name|String +43:23|index|[] +43:26|name|substrings +43:37|init|= +43:39|operator|new +43:43|name|String +43:49|index|[ +43:50|name|poss +43:54|operator|. +43:55|name|length +43:61|index|] +43:62|decl_stmt|; +44:17|if|if +44:20|condition|( +44:21|name|poss +44:25|operator|. +44:26|name|length +44:33|operator|>= +44:36|literal|1 +44:37|condition|) +44:39|block|{ +45:25|name|int +45:29|name|i +45:31|init|= +45:33|literal|0 +45:34|decl_stmt|; +46:25|name|substrings +46:35|index|[ +46:36|name|i +46:37|index|] +46:39|operator|= +46:41|name|str +46:44|operator|. +46:45|name|substring +46:54|argument_list|( +46:55|literal|0 +46:56|argument_list|, +46:58|name|poss +46:62|index|[ +46:63|literal|0 +46:64|index|] +46:65|argument_list|) +46:66|expr_stmt|; +47:25|operator|++ +47:27|name|i +47:28|expr_stmt|; +48:25|for|for +48:29|control|( +48:30|init|; +48:32|name|i +48:34|operator|< +48:36|name|poss +48:40|operator|. +48:41|name|length +48:47|condition|; +48:49|operator|++ +48:51|name|i +48:52|control|) +48:54|block|{ +49:33|name|substrings +49:43|index|[ +49:44|name|i +49:45|index|] +49:47|operator|= +49:49|name|str +49:52|operator|. +49:53|name|substring +49:62|argument_list|( +49:63|name|poss +49:67|index|[ +49:68|name|i +49:70|operator|- +49:72|literal|1 +49:73|index|] +49:75|operator|+ +49:77|literal|1 +49:78|argument_list|, +49:80|name|poss +49:84|index|[ +49:85|name|i +49:86|index|] +49:87|argument_list|) +49:88|expr_stmt|; +50:25|block|} +51:17|block|} +52:17|return|return +52:24|name|substrings +52:34|return|; +53:9|block|} +55:-|DECL|method|join (String[] ary, String with) +55:9|specifier|public +55:16|specifier|static +55:23|name|String +55:30|name|join +55:34|parameter_list|( +55:35|name|String +55:41|index|[] +55:44|name|ary +55:47|parameter_list|, +55:49|name|String +55:56|name|with +55:60|parameter_list|) +55:62|block|{ +56:17|name|StringBuffer +56:30|name|buf +56:34|init|= +56:36|operator|new +56:40|name|StringBuffer +56:52|argument_list|() +56:54|decl_stmt|; +57:17|for|for +57:21|control|( +57:22|name|int +57:26|name|i +57:28|init|= +57:30|literal|0 +57:31|init|; +57:33|name|i +57:35|operator|< +57:37|name|ary +57:40|operator|. +57:41|name|length +57:47|condition|; +57:49|operator|++ +57:51|name|i +57:52|control|) +57:54|block|{ +58:25|if|if +58:28|condition|( +58:29|name|i +58:31|operator|> +58:33|literal|0 +58:34|condition|) +58:36|block|{ +59:33|name|buf +59:36|operator|. +59:37|name|append +59:43|argument_list|( +59:44|name|with +59:48|argument_list|) +59:49|expr_stmt|; +60:25|block|} +61:25|name|buf +61:28|operator|. +61:29|name|append +61:35|argument_list|( +61:36|name|ary +61:39|index|[ +61:40|name|i +61:41|index|] +61:42|argument_list|) +61:43|expr_stmt|; +62:17|block|} +63:17|return|return +63:24|name|buf +63:27|operator|. +63:28|name|toString +63:36|argument_list|() +63:38|return|; +64:9|block|} +66:-|DECL|method|join (String[] ary, int begin, int end, String with) +66:9|specifier|public +66:16|specifier|static +66:23|name|String +66:30|name|join +66:34|parameter_list|( +66:35|name|String +66:41|index|[] +66:44|name|ary +66:47|parameter_list|, +66:49|name|int +66:53|name|begin +66:58|parameter_list|, +66:60|name|int +66:64|name|end +66:67|parameter_list|, +66:69|name|String +66:76|name|with +66:80|parameter_list|) +66:82|block|{ +67:17|if|if +67:20|condition|( +67:21|name|begin +67:27|operator|< +67:29|literal|0 +67:30|condition|) +67:32|block|{ +68:25|name|begin +68:31|operator|= +68:33|literal|0 +68:34|expr_stmt|; +69:17|block|} +70:17|if|if +70:20|condition|( +70:21|name|end +70:25|operator|> +70:27|name|ary +70:30|operator|. +70:31|name|length +70:37|condition|) +70:39|block|{ +71:25|name|end +71:29|operator|= +71:31|name|ary +71:34|operator|. +71:35|name|length +71:41|expr_stmt|; +72:17|block|} +73:17|name|StringBuffer +73:30|name|buf +73:34|init|= +73:36|operator|new +73:40|name|StringBuffer +73:52|argument_list|() +73:54|decl_stmt|; +74:17|for|for +74:21|control|( +74:22|name|int +74:26|name|i +74:28|init|= +74:30|name|begin +74:35|init|; +74:37|name|i +74:39|operator|< +74:41|name|end +74:44|condition|; +74:46|operator|++ +74:48|name|i +74:49|control|) +74:51|block|{ +75:25|if|if +75:28|condition|( +75:29|name|i +75:31|operator|> +75:33|name|begin +75:38|condition|) +75:40|block|{ +76:33|name|buf +76:36|operator|. +76:37|name|append +76:43|argument_list|( +76:44|name|with +76:48|argument_list|) +76:49|expr_stmt|; +77:25|block|} +78:25|name|buf +78:28|operator|. +78:29|name|append +78:35|argument_list|( +78:36|name|ary +78:39|index|[ +78:40|name|i +78:41|index|] +78:42|argument_list|) +78:43|expr_stmt|; +79:17|block|} +80:17|return|return +80:24|name|buf +80:27|operator|. +80:28|name|toString +80:36|argument_list|() +80:38|return|; +81:9|block|} +83:-|DECL|method|replaceFirst (String str, String pat, String replacement) +83:9|specifier|public +83:16|specifier|static +83:23|name|String +83:30|name|replaceFirst +83:42|parameter_list|( +83:43|name|String +83:50|name|str +83:53|parameter_list|, +83:55|name|String +83:62|name|pat +83:65|parameter_list|, +83:67|name|String +83:74|name|replacement +83:85|parameter_list|) +83:87|block|{ +84:17|name|int +84:21|name|pos +84:25|init|= +84:27|name|str +84:30|operator|. +84:31|name|indexOf +84:38|argument_list|( +84:39|name|pat +84:42|argument_list|) +84:43|decl_stmt|; +85:17|if|if +85:20|condition|( +85:21|name|pos +85:25|operator|>= +85:28|literal|0 +85:29|condition|) +85:31|block|{ +86:25|return|return +86:32|name|str +86:35|operator|. +86:36|name|substring +86:45|argument_list|( +86:46|literal|0 +86:47|argument_list|, +86:49|name|pos +86:52|argument_list|) +86:54|operator|+ +86:56|name|replacement +86:68|operator|+ +86:70|name|str +86:73|operator|. +86:74|name|substring +86:83|argument_list|( +86:84|name|pos +86:88|operator|+ +86:90|name|pat +86:93|operator|. +86:94|name|length +86:100|argument_list|() +86:102|argument_list|) +86:103|return|; +87:17|block|} +87:19|else|else +87:24|block|{ +88:25|return|return +88:32|name|str +88:35|return|; +89:17|block|} +90:9|block|} +92:1|comment|// public static Double[] scanDoubleValues(String str, int sepChar) { +93:1|comment|// String[] subs = StringUtil.split(str, sepChar); +94:1|comment|// Double[] values = new Double[subs.length]; +95:1|comment|// for (int i = 0; i< subs.length; ++i) { +96:1|comment|// try { +97:1|comment|// double v = Double.parseDouble(subs[i]); +98:1|comment|// values[i] = v; +99:1|comment|// } catch (NumberFormatException e) { +100:1|comment|// values[i] = null; +101:1|comment|// } +102:1|comment|// } +103:1|comment|// return values; +104:1|comment|// } +105:1|block|} +-:-|end_class +-:-| +-:-|end_unit +-:-| diff --git a/tokenize/t/expected/main.c.nopos.token b/tokenize/t/expected/main.c.nopos.token new file mode 100644 index 00000000..4d273d25 --- /dev/null +++ b/tokenize/t/expected/main.c.nopos.token @@ -0,0 +1,2070 @@ +begin_unit|revision:1.0.0;language:C;cregit-version:0.0.1 +begin_comment +comment|/* * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see. */ +end_comment + +begin_ifdef +ifdef|# +directive|ifdef +name|HAVE_CONFIG_H +end_ifdef + +begin_include +include|# +directive|include +file| +end_include + +begin_endif +endif|# +directive|endif +end_endif + +begin_include +include|# +directive|include +file| +end_include + +begin_include +include|# +directive|include +file| +end_include + +begin_include +include|# +directive|include +file| +end_include + +begin_include +include|# +directive|include +file| +end_include + +begin_include +include|# +directive|include +file| +end_include + +begin_include +include|# +directive|include +file|"xournal.h" +end_include + +begin_include +include|# +directive|include +file|"xo-interface.h" +end_include + +begin_include +include|# +directive|include +file|"xo-support.h" +end_include + +begin_include +include|# +directive|include +file|"xo-callbacks.h" +end_include + +begin_include +include|# +directive|include +file|"xo-misc.h" +end_include + +begin_include +include|# +directive|include +file|"xo-file.h" +end_include + +begin_include +include|# +directive|include +file|"xo-paint.h" +end_include + +begin_include +include|# +directive|include +file|"xo-shapes.h" +end_include + +begin_decl_stmt +DECL|variable|winMain +name|GtkWidget +modifier|* +name|winMain +decl_stmt|; +end_decl_stmt + +begin_decl_stmt +DECL|variable|canvas +name|GnomeCanvas +modifier|* +name|canvas +decl_stmt|; +end_decl_stmt + +begin_decl_stmt +DECL|variable|journal +name|struct +name|Journal +name|journal +decl_stmt|; +end_decl_stmt + +begin_comment +DECL|variable|journal +comment|// the journal +end_comment + +begin_decl_stmt +DECL|variable|bgpdf +name|struct +name|BgPdf +name|bgpdf +decl_stmt|; +end_decl_stmt + +begin_comment +DECL|variable|bgpdf +comment|// the PDF loader stuff +end_comment + +begin_decl_stmt +DECL|variable|ui +name|struct +name|UIData +name|ui +decl_stmt|; +end_decl_stmt + +begin_comment +DECL|variable|ui +comment|// the user interface data +end_comment + +begin_decl_stmt +DECL|variable|undo +DECL|variable|redo +name|struct +name|UndoItem +modifier|* +name|undo +decl_stmt|, +modifier|* +name|redo +decl_stmt|; +end_decl_stmt + +begin_comment +DECL|variable|undo +DECL|variable|redo +comment|// the undo and redo stacks +end_comment + +begin_decl_stmt +DECL|variable|DEFAULT_ZOOM +name|double +name|DEFAULT_ZOOM +decl_stmt|; +end_decl_stmt + +begin_function +DECL|function|init_stuff (int argc,char * argv[]) +name|void +name|init_stuff +parameter_list|( +name|int +name|argc +parameter_list|, +name|char +modifier|* +name|argv +index|[] +parameter_list|) +block|{ +name|GtkWidget +modifier|* +name|w +decl_stmt|; +name|GList +modifier|* +name|dev_list +decl_stmt|; +name|GdkDevice +modifier|* +name|device +decl_stmt|; +name|GdkScreen +modifier|* +name|screen +decl_stmt|; +name|int +name|i +decl_stmt|, +name|j +decl_stmt|; +name|struct +name|Brush +modifier|* +name|b +decl_stmt|; +name|gboolean +name|can_xinput +decl_stmt|, +name|success +decl_stmt|; +name|gchar +modifier|* +name|tmppath +decl_stmt|, +modifier|* +name|tmpfn +decl_stmt|; +comment|// create some data structures needed to populate the preferences +name|ui +operator|. +name|default_page +operator|. +name|bg +operator|= +name|g_new +argument_list|( +expr|struct +name|Background +argument_list|, +literal|1 +argument_list|) +expr_stmt|; +comment|// initialize config file names +name|tmppath +operator|= +name|g_build_filename +argument_list|( +name|g_get_home_dir +argument_list|() +argument_list|, +name|CONFIG_DIR +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_mkdir +argument_list|( +name|tmppath +argument_list|, +literal|0700 +argument_list|) +expr_stmt|; +comment|// safer (MRU data may be confidential) +name|ui +operator|. +name|mrufile +operator|= +name|g_build_filename +argument_list|( +name|tmppath +argument_list|, +name|MRU_FILE +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|ui +operator|. +name|configfile +operator|= +name|g_build_filename +argument_list|( +name|tmppath +argument_list|, +name|CONFIG_FILE +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_free +argument_list|( +name|tmppath +argument_list|) +expr_stmt|; +comment|// initialize preferences +name|init_config_default +argument_list|() +expr_stmt|; +name|load_config_from_file +argument_list|() +expr_stmt|; +name|ui +operator|. +name|font_name +operator|= +name|g_strdup +argument_list|( +name|ui +operator|. +name|default_font_name +argument_list|) +expr_stmt|; +name|ui +operator|. +name|font_size +operator|= +name|ui +operator|. +name|default_font_size +expr_stmt|; +name|ui +operator|. +name|hiliter_alpha_mask +operator|= +literal|0xffffff00 +operator|+ +operator|( +name|guint +operator|) +operator|( +literal|255 +operator|* +name|ui +operator|. +name|hiliter_opacity +operator|) +expr_stmt|; +comment|// we need an empty canvas prior to creating the journal structures +name|canvas +operator|= +name|GNOME_CANVAS +argument_list|( +name|gnome_canvas_new_aa +argument_list|() +argument_list|) +expr_stmt|; +comment|// initialize data +name|ui +operator|. +name|default_page +operator|. +name|bg +operator|-> +name|canvas_item +operator|= +name|NULL +expr_stmt|; +name|ui +operator|. +name|layerbox_length +operator|= +literal|0 +expr_stmt|; +if|if +condition|( +name|argc +operator|> +literal|2 +operator||| +operator|( +name|argc +operator|== +literal|2 +operator|&& +name|argv +index|[ +literal|1 +index|] +index|[ +literal|0 +index|] +operator|== +literal|'-' +operator|) +condition|) +block|{ +name|printf +argument_list|( +name|_ +argument_list|( +literal|"Invalid command line parameters.\n" +literal|"Usage: %s [filename.xoj]\n" +argument_list|) +argument_list|, +name|argv +index|[ +literal|0 +index|] +argument_list|) +expr_stmt|; +name|gtk_exit +argument_list|( +literal|0 +argument_list|) +expr_stmt|; +block|} +name|undo +operator|= +name|NULL +expr_stmt|; +name|redo +operator|= +name|NULL +expr_stmt|; +name|journal +operator|. +name|pages +operator|= +name|NULL +expr_stmt|; +name|bgpdf +operator|. +name|status +operator|= +name|STATUS_NOT_INIT +expr_stmt|; +name|new_journal +argument_list|() +expr_stmt|; +name|ui +operator|. +name|cur_item_type +operator|= +name|ITEM_NONE +expr_stmt|; +name|ui +operator|. +name|cur_item +operator|= +name|NULL +expr_stmt|; +name|ui +operator|. +name|cur_path +operator|. +name|coords +operator|= +name|NULL +expr_stmt|; +name|ui +operator|. +name|cur_path_storage_alloc +operator|= +literal|0 +expr_stmt|; +name|ui +operator|. +name|cur_path +operator|. +name|ref_count +operator|= +literal|1 +expr_stmt|; +name|ui +operator|. +name|cur_widths +operator|= +name|NULL +expr_stmt|; +name|ui +operator|. +name|cur_widths_storage_alloc +operator|= +literal|0 +expr_stmt|; +name|ui +operator|. +name|selection +operator|= +name|NULL +expr_stmt|; +name|ui +operator|. +name|cursor +operator|= +name|NULL +expr_stmt|; +name|ui +operator|. +name|pen_cursor_pix +operator|= +name|ui +operator|. +name|hiliter_cursor_pix +operator|= +name|NULL +expr_stmt|; +name|ui +operator|. +name|cur_brush +operator|= +operator|& +operator|( +name|ui +operator|. +name|brushes +index|[ +literal|0 +index|] +index|[ +name|ui +operator|. +name|toolno +index|[ +literal|0 +index|] +index|] +operator|) +expr_stmt|; +for|for +control|( +name|j +operator|= +literal|0 +init|; +name|j +operator|<= +name|NUM_BUTTONS +condition|; +name|j +operator|++ +control|) +for|for +control|( +name|i +operator|= +literal|0 +init|; +name|i +operator|< +name|NUM_STROKE_TOOLS +condition|; +name|i +operator|++ +control|) +block|{ +name|b +operator|= +operator|& +operator|( +name|ui +operator|. +name|brushes +index|[ +name|j +index|] +index|[ +name|i +index|] +operator|) +expr_stmt|; +name|b +operator|-> +name|tool_type +operator|= +name|i +expr_stmt|; +if|if +condition|( +name|b +operator|-> +name|color_no +operator|>= +literal|0 +condition|) +block|{ +name|b +operator|-> +name|color_rgba +operator|= +name|predef_colors_rgba +index|[ +name|b +operator|-> +name|color_no +index|] +expr_stmt|; +if|if +condition|( +name|i +operator|== +name|TOOL_HIGHLIGHTER +condition|) +block|{ +name|b +operator|-> +name|color_rgba +operator|&= +name|ui +operator|. +name|hiliter_alpha_mask +expr_stmt|; +block|} +block|} +name|b +operator|-> +name|thickness +operator|= +name|predef_thickness +index|[ +name|i +index|] +index|[ +name|b +operator|-> +name|thickness_no +index|] +expr_stmt|; +block|} +for|for +control|( +name|i +operator|= +literal|0 +init|; +name|i +operator|< +name|NUM_STROKE_TOOLS +condition|; +name|i +operator|++ +control|) +name|g_memmove +argument_list|( +name|ui +operator|. +name|default_brushes +operator|+ +name|i +argument_list|, +operator|& +operator|( +name|ui +operator|. +name|brushes +index|[ +literal|0 +index|] +index|[ +name|i +index|] +operator|) +argument_list|, +sizeof|sizeof +argument_list|( +expr|struct +name|Brush +argument_list|) +argument_list|) +expr_stmt|; +name|ui +operator|. +name|cur_mapping +operator|= +literal|0 +expr_stmt|; +name|ui +operator|. +name|which_unswitch_button +operator|= +literal|0 +expr_stmt|; +name|ui +operator|. +name|in_proximity +operator|= +name|FALSE +expr_stmt|; +name|ui +operator|. +name|warned_generate_fontconfig +operator|= +name|FALSE +expr_stmt|; +name|reset_recognizer +argument_list|() +expr_stmt|; +comment|// initialize various interface elements +name|gtk_window_set_default_size +argument_list|( +name|GTK_WINDOW +argument_list|( +name|winMain +argument_list|) +argument_list|, +name|ui +operator|. +name|window_default_width +argument_list|, +name|ui +operator|. +name|window_default_height +argument_list|) +expr_stmt|; +if|if +condition|( +name|ui +operator|. +name|maximize_at_start +condition|) +name|gtk_window_maximize +argument_list|( +name|GTK_WINDOW +argument_list|( +name|winMain +argument_list|) +argument_list|) +expr_stmt|; +name|update_toolbar_and_menu +argument_list|() +expr_stmt|; +name|update_font_button +argument_list|() +expr_stmt|; +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"journalApplyAllPages" +argument_list|) +argument_list|) +argument_list|, +name|ui +operator|. +name|bg_apply_all_pages +argument_list|) +expr_stmt|; +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"journalNewPageKeepsBG" +argument_list|) +argument_list|) +argument_list|, +name|ui +operator|. +name|new_page_bg_from_pdf +argument_list|) +expr_stmt|; +if|if +condition|( +name|ui +operator|. +name|fullscreen +condition|) +block|{ +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"viewFullscreen" +argument_list|) +argument_list|) +argument_list|, +name|TRUE +argument_list|) +expr_stmt|; +name|gtk_toggle_tool_button_set_active +argument_list|( +name|GTK_TOGGLE_TOOL_BUTTON +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"buttonFullscreen" +argument_list|) +argument_list|) +argument_list|, +name|TRUE +argument_list|) +expr_stmt|; +name|gtk_window_fullscreen +argument_list|( +name|GTK_WINDOW +argument_list|( +name|winMain +argument_list|) +argument_list|) +expr_stmt|; +block|} +name|gtk_button_set_relief +argument_list|( +name|GTK_BUTTON +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"buttonColorChooser" +argument_list|) +argument_list|) +argument_list|, +name|GTK_RELIEF_NONE +argument_list|) +expr_stmt|; +name|allow_all_accels +argument_list|() +expr_stmt|; +name|add_scroll_bindings +argument_list|() +expr_stmt|; +comment|// prevent interface items from stealing focus +comment|// glade doesn't properly handle can_focus, so manually set it +name|gtk_combo_box_set_focus_on_click +argument_list|( +name|GTK_COMBO_BOX +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"comboLayer" +argument_list|) +argument_list|) +argument_list|, +name|FALSE +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"spinPageNo" +argument_list|) +argument_list|, +literal|"activate" +argument_list|, +name|G_CALLBACK +argument_list|( +name|handle_activate_signal +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|gtk_container_forall +argument_list|( +name|GTK_CONTAINER +argument_list|( +name|winMain +argument_list|) +argument_list|, +name|unset_flags +argument_list|, +operator|( +name|gpointer +operator|) +name|GTK_CAN_FOCUS +argument_list|) +expr_stmt|; +name|GTK_WIDGET_SET_FLAGS +argument_list|( +name|GTK_WIDGET +argument_list|( +name|canvas +argument_list|) +argument_list|, +name|GTK_CAN_FOCUS +argument_list|) +expr_stmt|; +name|GTK_WIDGET_SET_FLAGS +argument_list|( +name|GTK_WIDGET +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"spinPageNo" +argument_list|) +argument_list|) +argument_list|, +name|GTK_CAN_FOCUS +argument_list|) +expr_stmt|; +comment|// install hooks on button/key/activation events to make the spinPageNo lose focus +name|gtk_container_forall +argument_list|( +name|GTK_CONTAINER +argument_list|( +name|winMain +argument_list|) +argument_list|, +name|install_focus_hooks +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +comment|// set up and initialize the canvas +name|gtk_widget_show +argument_list|( +name|GTK_WIDGET +argument_list|( +name|canvas +argument_list|) +argument_list|) +expr_stmt|; +name|w +operator|= +name|GET_COMPONENT +argument_list|( +literal|"scrolledwindowMain" +argument_list|) +expr_stmt|; +name|gtk_container_add +argument_list|( +name|GTK_CONTAINER +argument_list|( +name|w +argument_list|) +argument_list|, +name|GTK_WIDGET +argument_list|( +name|canvas +argument_list|) +argument_list|) +expr_stmt|; +name|gtk_scrolled_window_set_policy +argument_list|( +name|GTK_SCROLLED_WINDOW +argument_list|( +name|w +argument_list|) +argument_list|, +name|GTK_POLICY_AUTOMATIC +argument_list|, +name|GTK_POLICY_AUTOMATIC +argument_list|) +expr_stmt|; +name|gtk_widget_set_events +argument_list|( +name|GTK_WIDGET +argument_list|( +name|canvas +argument_list|) +argument_list|, +name|GDK_EXPOSURE_MASK +operator|| +name|GDK_POINTER_MOTION_MASK +operator|| +name|GDK_BUTTON_MOTION_MASK +operator|| +name|GDK_BUTTON_PRESS_MASK +operator|| +name|GDK_BUTTON_RELEASE_MASK +operator|| +name|GDK_KEY_PRESS_MASK +operator|| +name|GDK_ENTER_NOTIFY_MASK +operator|| +name|GDK_LEAVE_NOTIFY_MASK +operator|| +name|GDK_PROXIMITY_IN_MASK +operator|| +name|GDK_PROXIMITY_OUT_MASK +argument_list|) +expr_stmt|; +name|gnome_canvas_set_pixels_per_unit +argument_list|( +name|canvas +argument_list|, +name|ui +operator|. +name|zoom +argument_list|) +expr_stmt|; +name|gnome_canvas_set_center_scroll_region +argument_list|( +name|canvas +argument_list|, +name|TRUE +argument_list|) +expr_stmt|; +name|gtk_layout_get_hadjustment +argument_list|( +name|GTK_LAYOUT +argument_list|( +name|canvas +argument_list|) +argument_list|) +operator|-> +name|step_increment +operator|= +name|ui +operator|. +name|scrollbar_step_increment +expr_stmt|; +name|gtk_layout_get_vadjustment +argument_list|( +name|GTK_LAYOUT +argument_list|( +name|canvas +argument_list|) +argument_list|) +operator|-> +name|step_increment +operator|= +name|ui +operator|. +name|scrollbar_step_increment +expr_stmt|; +comment|// set up the page size and canvas size +name|update_page_stuff +argument_list|() +expr_stmt|; +name|g_signal_connect +argument_list|( +operator|( +name|gpointer +operator|) +name|canvas +argument_list|, +literal|"button_press_event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|on_canvas_button_press_event +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +operator|( +name|gpointer +operator|) +name|canvas +argument_list|, +literal|"button_release_event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|on_canvas_button_release_event +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +operator|( +name|gpointer +operator|) +name|canvas +argument_list|, +literal|"enter_notify_event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|on_canvas_enter_notify_event +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +operator|( +name|gpointer +operator|) +name|canvas +argument_list|, +literal|"leave_notify_event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|on_canvas_leave_notify_event +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +operator|( +name|gpointer +operator|) +name|canvas +argument_list|, +literal|"proximity_in_event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|on_canvas_proximity_event +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +operator|( +name|gpointer +operator|) +name|canvas +argument_list|, +literal|"proximity_out_event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|on_canvas_proximity_event +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +operator|( +name|gpointer +operator|) +name|canvas +argument_list|, +literal|"expose_event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|on_canvas_expose_event +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +operator|( +name|gpointer +operator|) +name|canvas +argument_list|, +literal|"key_press_event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|on_canvas_key_press_event +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +operator|( +name|gpointer +operator|) +name|canvas +argument_list|, +literal|"motion_notify_event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|on_canvas_motion_notify_event +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +operator|( +name|gpointer +operator|) +name|gtk_layout_get_vadjustment +argument_list|( +name|GTK_LAYOUT +argument_list|( +name|canvas +argument_list|) +argument_list|) +argument_list|, +literal|"value-changed" +argument_list|, +name|G_CALLBACK +argument_list|( +name|on_vscroll_changed +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +operator|( +name|gpointer +operator|) +name|gtk_layout_get_hadjustment +argument_list|( +name|GTK_LAYOUT +argument_list|( +name|canvas +argument_list|) +argument_list|) +argument_list|, +literal|"value-changed" +argument_list|, +name|G_CALLBACK +argument_list|( +name|on_hscroll_changed +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_object_set_data +argument_list|( +name|G_OBJECT +argument_list|( +name|winMain +argument_list|) +argument_list|, +literal|"canvas" +argument_list|, +name|canvas +argument_list|) +expr_stmt|; +name|screen +operator|= +name|gtk_widget_get_screen +argument_list|( +name|winMain +argument_list|) +expr_stmt|; +name|ui +operator|. +name|screen_width +operator|= +name|gdk_screen_get_width +argument_list|( +name|screen +argument_list|) +expr_stmt|; +name|ui +operator|. +name|screen_height +operator|= +name|gdk_screen_get_height +argument_list|( +name|screen +argument_list|) +expr_stmt|; +name|can_xinput +operator|= +name|FALSE +expr_stmt|; +name|dev_list +operator|= +name|gdk_devices_list +argument_list|() +expr_stmt|; +while|while +condition|( +name|dev_list +operator|!= +name|NULL +condition|) +block|{ +name|device +operator|= +operator|( +name|GdkDevice +operator|* +operator|) +name|dev_list +operator|-> +name|data +expr_stmt|; +if|if +condition|( +name|device +operator|!= +name|gdk_device_get_core_pointer +argument_list|() +operator|&& +name|device +operator|-> +name|num_axes +operator|>= +literal|2 +condition|) +block|{ +comment|/* get around a GDK bug: map the valuator range CORRECTLY to [0,1] */ +ifdef|# +directive|ifdef +name|ENABLE_XINPUT_BUGFIX +name|gdk_device_set_axis_use +argument_list|( +name|device +argument_list|, +literal|0 +argument_list|, +name|GDK_AXIS_IGNORE +argument_list|) +expr_stmt|; +name|gdk_device_set_axis_use +argument_list|( +name|device +argument_list|, +literal|1 +argument_list|, +name|GDK_AXIS_IGNORE +argument_list|) +expr_stmt|; +endif|# +directive|endif +name|gdk_device_set_mode +argument_list|( +name|device +argument_list|, +name|GDK_MODE_SCREEN +argument_list|) +expr_stmt|; +if|if +condition|( +name|g_strrstr +argument_list|( +name|device +operator|-> +name|name +argument_list|, +literal|"raser" +argument_list|) +condition|) +name|gdk_device_set_source +argument_list|( +name|device +argument_list|, +name|GDK_SOURCE_ERASER +argument_list|) +expr_stmt|; +name|can_xinput +operator|= +name|TRUE +expr_stmt|; +block|} +name|dev_list +operator|= +name|dev_list +operator|-> +name|next +expr_stmt|; +block|} +if|if +condition|( +operator|! +name|can_xinput +condition|) +name|gtk_widget_set_sensitive +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"optionsUseXInput" +argument_list|) +argument_list|, +name|FALSE +argument_list|) +expr_stmt|; +name|ui +operator|. +name|use_xinput +operator|= +name|ui +operator|. +name|allow_xinput +operator|&& +name|can_xinput +expr_stmt|; +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"optionsProgressiveBG" +argument_list|) +argument_list|) +argument_list|, +name|ui +operator|. +name|progressive_bg +argument_list|) +expr_stmt|; +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"optionsPrintRuling" +argument_list|) +argument_list|) +argument_list|, +name|ui +operator|. +name|print_ruling +argument_list|) +expr_stmt|; +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"optionsLegacyPDFExport" +argument_list|) +argument_list|) +argument_list|, +name|ui +operator|. +name|exportpdf_prefer_legacy +argument_list|) +expr_stmt|; +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"optionsLayersPDFExport" +argument_list|) +argument_list|) +argument_list|, +name|ui +operator|. +name|exportpdf_layers +argument_list|) +expr_stmt|; +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"optionsAutoloadPdfXoj" +argument_list|) +argument_list|) +argument_list|, +name|ui +operator|. +name|autoload_pdf_xoj +argument_list|) +expr_stmt|; +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"optionsAutosaveXoj" +argument_list|) +argument_list|) +argument_list|, +name|ui +operator|. +name|autosave_enabled +argument_list|) +expr_stmt|; +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"optionsLeftHanded" +argument_list|) +argument_list|) +argument_list|, +name|ui +operator|. +name|left_handed +argument_list|) +expr_stmt|; +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"optionsShortenMenus" +argument_list|) +argument_list|) +argument_list|, +name|ui +operator|. +name|shorten_menus +argument_list|) +expr_stmt|; +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"optionsAutoSavePrefs" +argument_list|) +argument_list|) +argument_list|, +name|ui +operator|. +name|auto_save_prefs +argument_list|) +expr_stmt|; +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"optionsButtonSwitchMapping" +argument_list|) +argument_list|) +argument_list|, +name|ui +operator|. +name|button_switch_mapping +argument_list|) +expr_stmt|; +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"optionsPenCursor" +argument_list|) +argument_list|) +argument_list|, +name|ui +operator|. +name|pen_cursor +argument_list|) +expr_stmt|; +name|hide_unimplemented +argument_list|() +expr_stmt|; +name|update_undo_redo_enabled +argument_list|() +expr_stmt|; +name|update_copy_paste_enabled +argument_list|() +expr_stmt|; +name|update_vbox_order +argument_list|( +name|ui +operator|. +name|vertical_order +index|[ +name|ui +operator|. +name|fullscreen +condition|? +literal|1 +else|: +literal|0 +index|] +argument_list|) +expr_stmt|; +name|gtk_widget_grab_focus +argument_list|( +name|GTK_WIDGET +argument_list|( +name|canvas +argument_list|) +argument_list|) +expr_stmt|; +comment|// show everything... +name|gtk_widget_show +argument_list|( +name|winMain +argument_list|) +expr_stmt|; +name|update_cursor +argument_list|() +expr_stmt|; +comment|/* this will cause extension events to get enabled/disabled, but we need the windows to be mapped first */ +name|gtk_check_menu_item_set_active +argument_list|( +name|GTK_CHECK_MENU_ITEM +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"optionsUseXInput" +argument_list|) +argument_list|) +argument_list|, +name|ui +operator|. +name|use_xinput +argument_list|) +expr_stmt|; +comment|/* fix a bug in GTK+ 2.16 and 2.17: scrollbars shouldn't get extended input events from pointer motion when cursor moves into main window */ +if|if +condition|( +operator|! +name|gtk_check_version +argument_list|( +literal|2 +argument_list|, +literal|16 +argument_list|, +literal|0 +argument_list|) +condition|) +block|{ +name|g_signal_connect +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"menubar" +argument_list|) +argument_list|, +literal|"event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|filter_extended_events +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"toolbarMain" +argument_list|) +argument_list|, +literal|"event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|filter_extended_events +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"toolbarPen" +argument_list|) +argument_list|, +literal|"event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|filter_extended_events +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +name|GET_COMPONENT +argument_list|( +literal|"statusbar" +argument_list|) +argument_list|, +literal|"event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|filter_extended_events +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +operator|( +name|gpointer +operator|) +operator|( +name|gtk_scrolled_window_get_vscrollbar +argument_list|( +name|GTK_SCROLLED_WINDOW +argument_list|( +name|w +argument_list|) +argument_list|) +operator|) +argument_list|, +literal|"event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|filter_extended_events +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_signal_connect +argument_list|( +operator|( +name|gpointer +operator|) +operator|( +name|gtk_scrolled_window_get_hscrollbar +argument_list|( +name|GTK_SCROLLED_WINDOW +argument_list|( +name|w +argument_list|) +argument_list|) +operator|) +argument_list|, +literal|"event" +argument_list|, +name|G_CALLBACK +argument_list|( +name|filter_extended_events +argument_list|) +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +block|} +comment|// load the MRU +name|init_mru +argument_list|() +expr_stmt|; +comment|// and finally, open a file specified on the command line +comment|// (moved here because display parameters weren't initialized yet...) +if|if +condition|( +name|argc +operator|== +literal|1 +condition|) +return|return; +name|set_cursor_busy +argument_list|( +name|TRUE +argument_list|) +expr_stmt|; +if|if +condition|( +name|g_path_is_absolute +argument_list|( +name|argv +index|[ +literal|1 +index|] +argument_list|) +condition|) +name|tmpfn +operator|= +name|g_strdup +argument_list|( +name|argv +index|[ +literal|1 +index|] +argument_list|) +expr_stmt|; +else|else +block|{ +name|tmppath +operator|= +name|g_get_current_dir +argument_list|() +expr_stmt|; +name|tmpfn +operator|= +name|g_build_filename +argument_list|( +name|tmppath +argument_list|, +name|argv +index|[ +literal|1 +index|] +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|g_free +argument_list|( +name|tmppath +argument_list|) +expr_stmt|; +block|} +name|success +operator|= +name|open_journal +argument_list|( +name|tmpfn +argument_list|) +expr_stmt|; +name|g_free +argument_list|( +name|tmpfn +argument_list|) +expr_stmt|; +name|set_cursor_busy +argument_list|( +name|FALSE +argument_list|) +expr_stmt|; +if|if +condition|( +operator|! +name|success +condition|) +block|{ +name|w +operator|= +name|gtk_message_dialog_new +argument_list|( +name|GTK_WINDOW +argument_list|( +name|winMain +argument_list|) +argument_list|, +name|GTK_DIALOG_DESTROY_WITH_PARENT +argument_list|, +name|GTK_MESSAGE_ERROR +argument_list|, +name|GTK_BUTTONS_OK +argument_list|, +name|_ +argument_list|( +literal|"Error opening file '%s'" +argument_list|) +argument_list|, +name|argv +index|[ +literal|1 +index|] +argument_list|) +expr_stmt|; +name|wrapper_gtk_dialog_run +argument_list|( +name|GTK_DIALOG +argument_list|( +name|w +argument_list|) +argument_list|) +expr_stmt|; +name|gtk_widget_destroy +argument_list|( +name|w +argument_list|) +expr_stmt|; +block|} +block|} +end_function + +begin_function +name|int +DECL|function|main (int argc,char * argv[]) +name|main +parameter_list|( +name|int +name|argc +parameter_list|, +name|char +modifier|* +name|argv +index|[] +parameter_list|) +block|{ +name|gchar +modifier|* +name|path +decl_stmt|, +modifier|* +name|path1 +decl_stmt|, +modifier|* +name|path2 +decl_stmt|; +ifdef|# +directive|ifdef +name|ENABLE_NLS +name|bindtextdomain +argument_list|( +name|GETTEXT_PACKAGE +argument_list|, +name|PACKAGE_LOCALE_DIR +argument_list|) +expr_stmt|; +name|bind_textdomain_codeset +argument_list|( +name|GETTEXT_PACKAGE +argument_list|, +literal|"UTF-8" +argument_list|) +expr_stmt|; +name|textdomain +argument_list|( +name|GETTEXT_PACKAGE +argument_list|) +expr_stmt|; +endif|# +directive|endif +name|gtk_set_locale +argument_list|() +expr_stmt|; +name|gtk_init +argument_list|( +operator|& +name|argc +argument_list|, +operator|& +name|argv +argument_list|) +expr_stmt|; +name|path +operator|= +name|g_path_get_dirname +argument_list|( +name|argv +index|[ +literal|0 +index|] +argument_list|) +expr_stmt|; +name|path1 +operator|= +name|g_build_filename +argument_list|( +name|path +argument_list|, +literal|"pixmaps" +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|path2 +operator|= +name|g_build_filename +argument_list|( +name|path +argument_list|, +literal|".." +argument_list|, +literal|"pixmaps" +argument_list|, +name|NULL +argument_list|) +expr_stmt|; +name|add_pixmap_directory +argument_list|( +name|path +argument_list|) +expr_stmt|; +name|add_pixmap_directory +argument_list|( +name|path2 +argument_list|) +expr_stmt|; +name|add_pixmap_directory +argument_list|( +name|path1 +argument_list|) +expr_stmt|; +name|g_free +argument_list|( +name|path +argument_list|) +expr_stmt|; +name|g_free +argument_list|( +name|path1 +argument_list|) +expr_stmt|; +name|g_free +argument_list|( +name|path2 +argument_list|) +expr_stmt|; +name|add_pixmap_directory +argument_list|( +name|PACKAGE_DATA_DIR +literal|"/" +name|PACKAGE +literal|"/pixmaps" +argument_list|) +expr_stmt|; +comment|/* * The following code was added by Glade to create one of each component * (except popup menus), just so that you see something after building * the project. Delete any components that you don't want shown initially. */ +name|winMain +operator|= +name|create_winMain +argument_list|() +expr_stmt|; +name|init_stuff +argument_list|( +name|argc +argument_list|, +name|argv +argument_list|) +expr_stmt|; +name|gtk_window_set_icon +argument_list|( +name|GTK_WINDOW +argument_list|( +name|winMain +argument_list|) +argument_list|, +name|create_pixbuf +argument_list|( +literal|"xournal.png" +argument_list|) +argument_list|) +expr_stmt|; +name|gtk_main +argument_list|() +expr_stmt|; +if|if +condition|( +name|bgpdf +operator|. +name|status +operator|!= +name|STATUS_NOT_INIT +condition|) +name|shutdown_bgpdf +argument_list|() +expr_stmt|; +name|save_mru_list +argument_list|() +expr_stmt|; +name|autosave_cleanup +argument_list|( +operator|& +name|ui +operator|. +name|autosave_filename_list +argument_list|) +expr_stmt|; +if|if +condition|( +name|ui +operator|. +name|auto_save_prefs +condition|) +name|save_config_to_file +argument_list|() +expr_stmt|; +return|return +literal|0 +return|; +block|} +end_function + +end_unit + diff --git a/tokenize/t/expected/main.c.token b/tokenize/t/expected/main.c.token new file mode 100644 index 00000000..1f329eda --- /dev/null +++ b/tokenize/t/expected/main.c.token @@ -0,0 +1,2070 @@ +-:-|begin_unit|revision:1.0.0;language:C;cregit-version:0.0.1 +-:-|begin_comment +1:1|comment|/* * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see. */ +-:-|end_comment +-:-| +-:-|begin_ifdef +16:1|ifdef|# +16:2|directive|ifdef +16:8|name|HAVE_CONFIG_H +-:-|end_ifdef +-:-| +-:-|begin_include +17:1|include|# +17:4|directive|include +17:12|file| +-:-|end_include +-:-| +-:-|begin_endif +18:1|endif|# +18:2|directive|endif +-:-|end_endif +-:-| +-:-|begin_include +20:1|include|# +20:2|directive|include +20:10|file| +-:-|end_include +-:-| +-:-|begin_include +21:1|include|# +21:2|directive|include +21:10|file| +-:-|end_include +-:-| +-:-|begin_include +22:1|include|# +22:2|directive|include +22:10|file| +-:-|end_include +-:-| +-:-|begin_include +23:1|include|# +23:2|directive|include +23:10|file| +-:-|end_include +-:-| +-:-|begin_include +24:1|include|# +24:2|directive|include +24:10|file| +-:-|end_include +-:-| +-:-|begin_include +26:1|include|# +26:2|directive|include +26:10|file|"xournal.h" +-:-|end_include +-:-| +-:-|begin_include +27:1|include|# +27:2|directive|include +27:10|file|"xo-interface.h" +-:-|end_include +-:-| +-:-|begin_include +28:1|include|# +28:2|directive|include +28:10|file|"xo-support.h" +-:-|end_include +-:-| +-:-|begin_include +29:1|include|# +29:2|directive|include +29:10|file|"xo-callbacks.h" +-:-|end_include +-:-| +-:-|begin_include +30:1|include|# +30:2|directive|include +30:10|file|"xo-misc.h" +-:-|end_include +-:-| +-:-|begin_include +31:1|include|# +31:2|directive|include +31:10|file|"xo-file.h" +-:-|end_include +-:-| +-:-|begin_include +32:1|include|# +32:2|directive|include +32:10|file|"xo-paint.h" +-:-|end_include +-:-| +-:-|begin_include +33:1|include|# +33:2|directive|include +33:10|file|"xo-shapes.h" +-:-|end_include +-:-| +-:-|begin_decl_stmt +35:-|DECL|variable|winMain +35:1|name|GtkWidget +35:11|modifier|* +35:12|name|winMain +35:19|decl_stmt|; +-:-|end_decl_stmt +-:-| +-:-|begin_decl_stmt +36:-|DECL|variable|canvas +36:1|name|GnomeCanvas +36:13|modifier|* +36:14|name|canvas +36:20|decl_stmt|; +-:-|end_decl_stmt +-:-| +-:-|begin_decl_stmt +38:-|DECL|variable|journal +38:1|name|struct +38:8|name|Journal +38:16|name|journal +38:23|decl_stmt|; +-:-|end_decl_stmt +-:-| +-:-|begin_comment +38:-|DECL|variable|journal +38:25|comment|// the journal +-:-|end_comment +-:-| +-:-|begin_decl_stmt +39:-|DECL|variable|bgpdf +39:1|name|struct +39:8|name|BgPdf +39:14|name|bgpdf +39:19|decl_stmt|; +-:-|end_decl_stmt +-:-| +-:-|begin_comment +39:-|DECL|variable|bgpdf +39:22|comment|// the PDF loader stuff +-:-|end_comment +-:-| +-:-|begin_decl_stmt +40:-|DECL|variable|ui +40:1|name|struct +40:8|name|UIData +40:15|name|ui +40:17|decl_stmt|; +-:-|end_decl_stmt +-:-| +-:-|begin_comment +40:-|DECL|variable|ui +40:21|comment|// the user interface data +-:-|end_comment +-:-| +-:-|begin_decl_stmt +41:-|DECL|variable|undo +41:-|DECL|variable|redo +41:1|name|struct +41:8|name|UndoItem +41:17|modifier|* +41:18|name|undo +41:22|decl_stmt|, +41:24|modifier|* +41:25|name|redo +41:29|decl_stmt|; +-:-|end_decl_stmt +-:-| +-:-|begin_comment +41:-|DECL|variable|undo +41:-|DECL|variable|redo +41:31|comment|// the undo and redo stacks +-:-|end_comment +-:-| +-:-|begin_decl_stmt +43:-|DECL|variable|DEFAULT_ZOOM +43:1|name|double +43:8|name|DEFAULT_ZOOM +43:20|decl_stmt|; +-:-|end_decl_stmt +-:-| +-:-|begin_function +45:-|DECL|function|init_stuff (int argc,char * argv[]) +45:1|name|void +45:6|name|init_stuff +45:17|parameter_list|( +45:18|name|int +45:22|name|argc +45:26|parameter_list|, +45:28|name|char +45:33|modifier|* +45:34|name|argv +45:38|index|[] +45:40|parameter_list|) +46:1|block|{ +47:3|name|GtkWidget +47:13|modifier|* +47:14|name|w +47:15|decl_stmt|; +48:3|name|GList +48:9|modifier|* +48:10|name|dev_list +48:18|decl_stmt|; +49:3|name|GdkDevice +49:13|modifier|* +49:14|name|device +49:20|decl_stmt|; +50:3|name|GdkScreen +50:13|modifier|* +50:14|name|screen +50:20|decl_stmt|; +51:3|name|int +51:7|name|i +51:8|decl_stmt|, +51:10|name|j +51:11|decl_stmt|; +52:3|name|struct +52:10|name|Brush +52:16|modifier|* +52:17|name|b +52:18|decl_stmt|; +53:3|name|gboolean +53:12|name|can_xinput +53:22|decl_stmt|, +53:24|name|success +53:31|decl_stmt|; +54:3|name|gchar +54:9|modifier|* +54:10|name|tmppath +54:17|decl_stmt|, +54:19|modifier|* +54:20|name|tmpfn +54:25|decl_stmt|; +56:3|comment|// create some data structures needed to populate the preferences +57:3|name|ui +57:5|operator|. +57:6|name|default_page +57:18|operator|. +57:19|name|bg +57:22|operator|= +57:24|name|g_new +57:29|argument_list|( +57:30|expr|struct +57:37|name|Background +57:47|argument_list|, +57:49|literal|1 +57:50|argument_list|) +57:51|expr_stmt|; +59:3|comment|// initialize config file names +60:3|name|tmppath +60:11|operator|= +60:13|name|g_build_filename +60:29|argument_list|( +60:30|name|g_get_home_dir +60:44|argument_list|() +60:46|argument_list|, +60:48|name|CONFIG_DIR +60:58|argument_list|, +60:60|name|NULL +60:64|argument_list|) +60:65|expr_stmt|; +61:3|name|g_mkdir +61:10|argument_list|( +61:11|name|tmppath +61:18|argument_list|, +61:20|literal|0700 +61:24|argument_list|) +61:25|expr_stmt|; +61:27|comment|// safer (MRU data may be confidential) +62:3|name|ui +62:5|operator|. +62:6|name|mrufile +62:14|operator|= +62:16|name|g_build_filename +62:32|argument_list|( +62:33|name|tmppath +62:40|argument_list|, +62:42|name|MRU_FILE +62:50|argument_list|, +62:52|name|NULL +62:56|argument_list|) +62:57|expr_stmt|; +63:3|name|ui +63:5|operator|. +63:6|name|configfile +63:17|operator|= +63:19|name|g_build_filename +63:35|argument_list|( +63:36|name|tmppath +63:43|argument_list|, +63:45|name|CONFIG_FILE +63:56|argument_list|, +63:58|name|NULL +63:62|argument_list|) +63:63|expr_stmt|; +64:3|name|g_free +64:9|argument_list|( +64:10|name|tmppath +64:17|argument_list|) +64:18|expr_stmt|; +66:3|comment|// initialize preferences +67:3|name|init_config_default +67:22|argument_list|() +67:24|expr_stmt|; +68:3|name|load_config_from_file +68:24|argument_list|() +68:26|expr_stmt|; +69:3|name|ui +69:5|operator|. +69:6|name|font_name +69:16|operator|= +69:18|name|g_strdup +69:26|argument_list|( +69:27|name|ui +69:29|operator|. +69:30|name|default_font_name +69:47|argument_list|) +69:48|expr_stmt|; +70:3|name|ui +70:5|operator|. +70:6|name|font_size +70:16|operator|= +70:18|name|ui +70:20|operator|. +70:21|name|default_font_size +70:38|expr_stmt|; +71:3|name|ui +71:5|operator|. +71:6|name|hiliter_alpha_mask +71:25|operator|= +71:27|literal|0xffffff00 +71:38|operator|+ +71:40|operator|( +71:41|name|guint +71:46|operator|) +71:47|operator|( +71:48|literal|255 +71:51|operator|* +71:52|name|ui +71:54|operator|. +71:55|name|hiliter_opacity +71:70|operator|) +71:71|expr_stmt|; +73:3|comment|// we need an empty canvas prior to creating the journal structures +74:3|name|canvas +74:10|operator|= +74:12|name|GNOME_CANVAS +74:25|argument_list|( +74:26|name|gnome_canvas_new_aa +74:46|argument_list|() +74:48|argument_list|) +74:49|expr_stmt|; +76:3|comment|// initialize data +77:3|name|ui +77:5|operator|. +77:6|name|default_page +77:18|operator|. +77:19|name|bg +77:21|operator|-> +77:23|name|canvas_item +77:35|operator|= +77:37|name|NULL +77:41|expr_stmt|; +78:3|name|ui +78:5|operator|. +78:6|name|layerbox_length +78:22|operator|= +78:24|literal|0 +78:25|expr_stmt|; +80:3|if|if +80:6|condition|( +80:7|name|argc +80:12|operator|> +80:14|literal|2 +80:16|operator||| +80:19|operator|( +80:20|name|argc +80:25|operator|== +80:28|literal|2 +80:30|operator|&& +80:33|name|argv +80:37|index|[ +80:38|literal|1 +80:39|index|] +80:40|index|[ +80:41|literal|0 +80:42|index|] +80:44|operator|== +80:47|literal|'-' +80:50|operator|) +80:51|condition|) +80:53|block|{ +81:5|name|printf +81:11|argument_list|( +81:12|name|_ +81:13|argument_list|( +81:14|literal|"Invalid command line parameters.\n" +82:12|literal|"Usage: %s [filename.xoj]\n" +82:40|argument_list|) +82:41|argument_list|, +82:43|name|argv +82:47|index|[ +82:48|literal|0 +82:49|index|] +82:50|argument_list|) +82:51|expr_stmt|; +83:5|name|gtk_exit +83:13|argument_list|( +83:14|literal|0 +83:15|argument_list|) +83:16|expr_stmt|; +84:3|block|} +86:3|name|undo +86:8|operator|= +86:10|name|NULL +86:14|expr_stmt|; +86:16|name|redo +86:21|operator|= +86:23|name|NULL +86:27|expr_stmt|; +87:3|name|journal +87:10|operator|. +87:11|name|pages +87:17|operator|= +87:19|name|NULL +87:23|expr_stmt|; +88:3|name|bgpdf +88:8|operator|. +88:9|name|status +88:16|operator|= +88:18|name|STATUS_NOT_INIT +88:33|expr_stmt|; +90:3|name|new_journal +90:14|argument_list|() +90:16|expr_stmt|; +92:3|name|ui +92:5|operator|. +92:6|name|cur_item_type +92:20|operator|= +92:22|name|ITEM_NONE +92:31|expr_stmt|; +93:3|name|ui +93:5|operator|. +93:6|name|cur_item +93:15|operator|= +93:17|name|NULL +93:21|expr_stmt|; +94:3|name|ui +94:5|operator|. +94:6|name|cur_path +94:14|operator|. +94:15|name|coords +94:22|operator|= +94:24|name|NULL +94:28|expr_stmt|; +95:3|name|ui +95:5|operator|. +95:6|name|cur_path_storage_alloc +95:29|operator|= +95:31|literal|0 +95:32|expr_stmt|; +96:3|name|ui +96:5|operator|. +96:6|name|cur_path +96:14|operator|. +96:15|name|ref_count +96:25|operator|= +96:27|literal|1 +96:28|expr_stmt|; +97:3|name|ui +97:5|operator|. +97:6|name|cur_widths +97:17|operator|= +97:19|name|NULL +97:23|expr_stmt|; +98:3|name|ui +98:5|operator|. +98:6|name|cur_widths_storage_alloc +98:31|operator|= +98:33|literal|0 +98:34|expr_stmt|; +100:3|name|ui +100:5|operator|. +100:6|name|selection +100:16|operator|= +100:18|name|NULL +100:22|expr_stmt|; +101:3|name|ui +101:5|operator|. +101:6|name|cursor +101:13|operator|= +101:15|name|NULL +101:19|expr_stmt|; +102:3|name|ui +102:5|operator|. +102:6|name|pen_cursor_pix +102:21|operator|= +102:23|name|ui +102:25|operator|. +102:26|name|hiliter_cursor_pix +102:45|operator|= +102:47|name|NULL +102:51|expr_stmt|; +104:3|name|ui +104:5|operator|. +104:6|name|cur_brush +104:16|operator|= +104:18|operator|& +104:19|operator|( +104:20|name|ui +104:22|operator|. +104:23|name|brushes +104:30|index|[ +104:31|literal|0 +104:32|index|] +104:33|index|[ +104:34|name|ui +104:36|operator|. +104:37|name|toolno +104:43|index|[ +104:44|literal|0 +104:45|index|] +104:46|index|] +104:47|operator|) +104:48|expr_stmt|; +105:3|for|for +105:7|control|( +105:8|name|j +105:9|operator|= +105:10|literal|0 +105:11|init|; +105:13|name|j +105:14|operator|<= +105:16|name|NUM_BUTTONS +105:27|condition|; +105:29|name|j +105:30|operator|++ +105:32|control|) +106:5|for|for +106:9|control|( +106:10|name|i +106:11|operator|= +106:12|literal|0 +106:13|init|; +106:15|name|i +106:17|operator|< +106:19|name|NUM_STROKE_TOOLS +106:35|condition|; +106:37|name|i +106:38|operator|++ +106:40|control|) +106:42|block|{ +107:7|name|b +107:9|operator|= +107:11|operator|& +107:12|operator|( +107:13|name|ui +107:15|operator|. +107:16|name|brushes +107:23|index|[ +107:24|name|j +107:25|index|] +107:26|index|[ +107:27|name|i +107:28|index|] +107:29|operator|) +107:30|expr_stmt|; +108:7|name|b +108:8|operator|-> +108:10|name|tool_type +108:20|operator|= +108:22|name|i +108:23|expr_stmt|; +109:7|if|if +109:10|condition|( +109:11|name|b +109:12|operator|-> +109:14|name|color_no +109:22|operator|>= +109:24|literal|0 +109:25|condition|) +109:27|block|{ +110:9|name|b +110:10|operator|-> +110:12|name|color_rgba +110:23|operator|= +110:25|name|predef_colors_rgba +110:43|index|[ +110:44|name|b +110:45|operator|-> +110:47|name|color_no +110:55|index|] +110:56|expr_stmt|; +111:9|if|if +111:12|condition|( +111:13|name|i +111:15|operator|== +111:18|name|TOOL_HIGHLIGHTER +111:34|condition|) +111:36|block|{ +112:11|name|b +112:12|operator|-> +112:14|name|color_rgba +112:25|operator|&= +112:28|name|ui +112:30|operator|. +112:31|name|hiliter_alpha_mask +112:49|expr_stmt|; +113:9|block|} +114:7|block|} +115:7|name|b +115:8|operator|-> +115:10|name|thickness +115:20|operator|= +115:22|name|predef_thickness +115:38|index|[ +115:39|name|i +115:40|index|] +115:41|index|[ +115:42|name|b +115:43|operator|-> +115:45|name|thickness_no +115:57|index|] +115:58|expr_stmt|; +116:5|block|} +117:3|for|for +117:7|control|( +117:8|name|i +117:9|operator|= +117:10|literal|0 +117:11|init|; +117:13|name|i +117:14|operator|< +117:15|name|NUM_STROKE_TOOLS +117:31|condition|; +117:33|name|i +117:34|operator|++ +117:36|control|) +118:5|name|g_memmove +118:14|argument_list|( +118:15|name|ui +118:17|operator|. +118:18|name|default_brushes +118:33|operator|+ +118:34|name|i +118:35|argument_list|, +118:37|operator|& +118:38|operator|( +118:39|name|ui +118:41|operator|. +118:42|name|brushes +118:49|index|[ +118:50|literal|0 +118:51|index|] +118:52|index|[ +118:53|name|i +118:54|index|] +118:55|operator|) +118:56|argument_list|, +118:58|sizeof|sizeof +118:64|argument_list|( +118:65|expr|struct +118:72|name|Brush +118:77|argument_list|) +118:78|argument_list|) +118:79|expr_stmt|; +120:3|name|ui +120:5|operator|. +120:6|name|cur_mapping +120:18|operator|= +120:20|literal|0 +120:21|expr_stmt|; +121:3|name|ui +121:5|operator|. +121:6|name|which_unswitch_button +121:28|operator|= +121:30|literal|0 +121:31|expr_stmt|; +122:3|name|ui +122:5|operator|. +122:6|name|in_proximity +122:19|operator|= +122:21|name|FALSE +122:26|expr_stmt|; +123:3|name|ui +123:5|operator|. +123:6|name|warned_generate_fontconfig +123:33|operator|= +123:35|name|FALSE +123:40|expr_stmt|; +125:3|name|reset_recognizer +125:19|argument_list|() +125:21|expr_stmt|; +127:3|comment|// initialize various interface elements +129:3|name|gtk_window_set_default_size +129:30|argument_list|( +129:31|name|GTK_WINDOW +129:42|argument_list|( +129:43|name|winMain +129:50|argument_list|) +129:51|argument_list|, +129:53|name|ui +129:55|operator|. +129:56|name|window_default_width +129:76|argument_list|, +129:78|name|ui +129:80|operator|. +129:81|name|window_default_height +129:102|argument_list|) +129:103|expr_stmt|; +130:3|if|if +130:6|condition|( +130:7|name|ui +130:9|operator|. +130:10|name|maximize_at_start +130:27|condition|) +130:29|name|gtk_window_maximize +130:48|argument_list|( +130:49|name|GTK_WINDOW +130:60|argument_list|( +130:61|name|winMain +130:68|argument_list|) +130:69|argument_list|) +130:70|expr_stmt|; +131:3|name|update_toolbar_and_menu +131:26|argument_list|() +131:28|expr_stmt|; +132:3|name|update_font_button +132:21|argument_list|() +132:23|expr_stmt|; +134:3|name|gtk_check_menu_item_set_active +134:33|argument_list|( +135:5|name|GTK_CHECK_MENU_ITEM +135:24|argument_list|( +135:25|name|GET_COMPONENT +135:38|argument_list|( +135:39|literal|"journalApplyAllPages" +135:61|argument_list|) +135:62|argument_list|) +135:63|argument_list|, +135:65|name|ui +135:67|operator|. +135:68|name|bg_apply_all_pages +135:86|argument_list|) +135:87|expr_stmt|; +136:3|name|gtk_check_menu_item_set_active +136:33|argument_list|( +137:5|name|GTK_CHECK_MENU_ITEM +137:24|argument_list|( +137:25|name|GET_COMPONENT +137:38|argument_list|( +137:39|literal|"journalNewPageKeepsBG" +137:62|argument_list|) +137:63|argument_list|) +137:64|argument_list|, +137:66|name|ui +137:68|operator|. +137:69|name|new_page_bg_from_pdf +137:89|argument_list|) +137:90|expr_stmt|; +138:3|if|if +138:6|condition|( +138:7|name|ui +138:9|operator|. +138:10|name|fullscreen +138:20|condition|) +138:22|block|{ +139:5|name|gtk_check_menu_item_set_active +139:35|argument_list|( +140:7|name|GTK_CHECK_MENU_ITEM +140:26|argument_list|( +140:27|name|GET_COMPONENT +140:40|argument_list|( +140:41|literal|"viewFullscreen" +140:57|argument_list|) +140:58|argument_list|) +140:59|argument_list|, +140:61|name|TRUE +140:65|argument_list|) +140:66|expr_stmt|; +141:5|name|gtk_toggle_tool_button_set_active +141:38|argument_list|( +142:7|name|GTK_TOGGLE_TOOL_BUTTON +142:29|argument_list|( +142:30|name|GET_COMPONENT +142:43|argument_list|( +142:44|literal|"buttonFullscreen" +142:62|argument_list|) +142:63|argument_list|) +142:64|argument_list|, +142:66|name|TRUE +142:70|argument_list|) +142:71|expr_stmt|; +143:5|name|gtk_window_fullscreen +143:26|argument_list|( +143:27|name|GTK_WINDOW +143:37|argument_list|( +143:38|name|winMain +143:45|argument_list|) +143:46|argument_list|) +143:47|expr_stmt|; +144:3|block|} +145:3|name|gtk_button_set_relief +145:24|argument_list|( +145:25|name|GTK_BUTTON +145:35|argument_list|( +145:36|name|GET_COMPONENT +145:49|argument_list|( +145:50|literal|"buttonColorChooser" +145:70|argument_list|) +145:71|argument_list|) +145:72|argument_list|, +145:74|name|GTK_RELIEF_NONE +145:89|argument_list|) +145:90|expr_stmt|; +147:3|name|allow_all_accels +147:19|argument_list|() +147:21|expr_stmt|; +148:3|name|add_scroll_bindings +148:22|argument_list|() +148:24|expr_stmt|; +150:3|comment|// prevent interface items from stealing focus +151:3|comment|// glade doesn't properly handle can_focus, so manually set it +152:3|name|gtk_combo_box_set_focus_on_click +152:35|argument_list|( +152:36|name|GTK_COMBO_BOX +152:49|argument_list|( +152:50|name|GET_COMPONENT +152:63|argument_list|( +152:64|literal|"comboLayer" +152:76|argument_list|) +152:77|argument_list|) +152:78|argument_list|, +152:80|name|FALSE +152:85|argument_list|) +152:86|expr_stmt|; +153:3|name|g_signal_connect +153:19|argument_list|( +153:20|name|GET_COMPONENT +153:33|argument_list|( +153:34|literal|"spinPageNo" +153:46|argument_list|) +153:47|argument_list|, +153:49|literal|"activate" +153:59|argument_list|, +154:11|name|G_CALLBACK +154:21|argument_list|( +154:22|name|handle_activate_signal +154:44|argument_list|) +154:45|argument_list|, +154:47|name|NULL +154:51|argument_list|) +154:52|expr_stmt|; +155:3|name|gtk_container_forall +155:23|argument_list|( +155:24|name|GTK_CONTAINER +155:37|argument_list|( +155:38|name|winMain +155:45|argument_list|) +155:46|argument_list|, +155:48|name|unset_flags +155:59|argument_list|, +155:61|operator|( +155:62|name|gpointer +155:70|operator|) +155:71|name|GTK_CAN_FOCUS +155:84|argument_list|) +155:85|expr_stmt|; +156:3|name|GTK_WIDGET_SET_FLAGS +156:23|argument_list|( +156:24|name|GTK_WIDGET +156:34|argument_list|( +156:35|name|canvas +156:41|argument_list|) +156:42|argument_list|, +156:44|name|GTK_CAN_FOCUS +156:57|argument_list|) +156:58|expr_stmt|; +157:3|name|GTK_WIDGET_SET_FLAGS +157:23|argument_list|( +157:24|name|GTK_WIDGET +157:34|argument_list|( +157:35|name|GET_COMPONENT +157:48|argument_list|( +157:49|literal|"spinPageNo" +157:61|argument_list|) +157:62|argument_list|) +157:63|argument_list|, +157:65|name|GTK_CAN_FOCUS +157:78|argument_list|) +157:79|expr_stmt|; +159:3|comment|// install hooks on button/key/activation events to make the spinPageNo lose focus +160:3|name|gtk_container_forall +160:23|argument_list|( +160:24|name|GTK_CONTAINER +160:37|argument_list|( +160:38|name|winMain +160:45|argument_list|) +160:46|argument_list|, +160:48|name|install_focus_hooks +160:67|argument_list|, +160:69|name|NULL +160:73|argument_list|) +160:74|expr_stmt|; +162:3|comment|// set up and initialize the canvas +164:3|name|gtk_widget_show +164:19|argument_list|( +164:20|name|GTK_WIDGET +164:31|argument_list|( +164:32|name|canvas +164:38|argument_list|) +164:39|argument_list|) +164:40|expr_stmt|; +165:3|name|w +165:5|operator|= +165:7|name|GET_COMPONENT +165:20|argument_list|( +165:21|literal|"scrolledwindowMain" +165:41|argument_list|) +165:42|expr_stmt|; +166:3|name|gtk_container_add +166:21|argument_list|( +166:22|name|GTK_CONTAINER +166:36|argument_list|( +166:37|name|w +166:38|argument_list|) +166:39|argument_list|, +166:41|name|GTK_WIDGET +166:52|argument_list|( +166:53|name|canvas +166:59|argument_list|) +166:60|argument_list|) +166:61|expr_stmt|; +167:3|name|gtk_scrolled_window_set_policy +167:33|argument_list|( +167:34|name|GTK_SCROLLED_WINDOW +167:54|argument_list|( +167:55|name|w +167:56|argument_list|) +167:57|argument_list|, +167:59|name|GTK_POLICY_AUTOMATIC +167:79|argument_list|, +167:81|name|GTK_POLICY_AUTOMATIC +167:101|argument_list|) +167:102|expr_stmt|; +168:3|name|gtk_widget_set_events +168:25|argument_list|( +168:26|name|GTK_WIDGET +168:37|argument_list|( +168:38|name|canvas +168:44|argument_list|) +168:45|argument_list|, +169:6|name|GDK_EXPOSURE_MASK +169:24|operator|| +169:26|name|GDK_POINTER_MOTION_MASK +169:50|operator|| +169:52|name|GDK_BUTTON_MOTION_MASK +169:75|operator|| +170:6|name|GDK_BUTTON_PRESS_MASK +170:28|operator|| +170:30|name|GDK_BUTTON_RELEASE_MASK +170:54|operator|| +170:56|name|GDK_KEY_PRESS_MASK +170:75|operator|| +171:6|name|GDK_ENTER_NOTIFY_MASK +171:28|operator|| +171:30|name|GDK_LEAVE_NOTIFY_MASK +171:52|operator|| +172:6|name|GDK_PROXIMITY_IN_MASK +172:28|operator|| +172:30|name|GDK_PROXIMITY_OUT_MASK +172:52|argument_list|) +172:53|expr_stmt|; +173:3|name|gnome_canvas_set_pixels_per_unit +173:36|argument_list|( +173:37|name|canvas +173:43|argument_list|, +173:45|name|ui +173:47|operator|. +173:48|name|zoom +173:52|argument_list|) +173:53|expr_stmt|; +174:3|name|gnome_canvas_set_center_scroll_region +174:41|argument_list|( +174:42|name|canvas +174:48|argument_list|, +174:50|name|TRUE +174:54|argument_list|) +174:55|expr_stmt|; +175:3|name|gtk_layout_get_hadjustment +175:29|argument_list|( +175:30|name|GTK_LAYOUT +175:41|argument_list|( +175:42|name|canvas +175:48|argument_list|) +175:49|argument_list|) +175:50|operator|-> +175:52|name|step_increment +175:67|operator|= +175:69|name|ui +175:71|operator|. +175:72|name|scrollbar_step_increment +175:96|expr_stmt|; +176:3|name|gtk_layout_get_vadjustment +176:29|argument_list|( +176:30|name|GTK_LAYOUT +176:41|argument_list|( +176:42|name|canvas +176:48|argument_list|) +176:49|argument_list|) +176:50|operator|-> +176:52|name|step_increment +176:67|operator|= +176:69|name|ui +176:71|operator|. +176:72|name|scrollbar_step_increment +176:96|expr_stmt|; +178:3|comment|// set up the page size and canvas size +179:3|name|update_page_stuff +179:20|argument_list|() +179:22|expr_stmt|; +181:3|name|g_signal_connect +181:20|argument_list|( +181:21|operator|( +181:22|name|gpointer +181:30|operator|) +181:32|name|canvas +181:38|argument_list|, +181:40|literal|"button_press_event" +181:60|argument_list|, +182:21|name|G_CALLBACK +182:32|argument_list|( +182:33|name|on_canvas_button_press_event +182:61|argument_list|) +182:62|argument_list|, +183:21|name|NULL +183:25|argument_list|) +183:26|expr_stmt|; +184:3|name|g_signal_connect +184:20|argument_list|( +184:21|operator|( +184:22|name|gpointer +184:30|operator|) +184:32|name|canvas +184:38|argument_list|, +184:40|literal|"button_release_event" +184:62|argument_list|, +185:21|name|G_CALLBACK +185:32|argument_list|( +185:33|name|on_canvas_button_release_event +185:63|argument_list|) +185:64|argument_list|, +186:21|name|NULL +186:25|argument_list|) +186:26|expr_stmt|; +187:3|name|g_signal_connect +187:20|argument_list|( +187:21|operator|( +187:22|name|gpointer +187:30|operator|) +187:32|name|canvas +187:38|argument_list|, +187:40|literal|"enter_notify_event" +187:60|argument_list|, +188:21|name|G_CALLBACK +188:32|argument_list|( +188:33|name|on_canvas_enter_notify_event +188:61|argument_list|) +188:62|argument_list|, +189:21|name|NULL +189:25|argument_list|) +189:26|expr_stmt|; +190:3|name|g_signal_connect +190:20|argument_list|( +190:21|operator|( +190:22|name|gpointer +190:30|operator|) +190:32|name|canvas +190:38|argument_list|, +190:40|literal|"leave_notify_event" +190:60|argument_list|, +191:21|name|G_CALLBACK +191:32|argument_list|( +191:33|name|on_canvas_leave_notify_event +191:61|argument_list|) +191:62|argument_list|, +192:21|name|NULL +192:25|argument_list|) +192:26|expr_stmt|; +193:3|name|g_signal_connect +193:20|argument_list|( +193:21|operator|( +193:22|name|gpointer +193:30|operator|) +193:32|name|canvas +193:38|argument_list|, +193:40|literal|"proximity_in_event" +193:60|argument_list|, +194:21|name|G_CALLBACK +194:32|argument_list|( +194:33|name|on_canvas_proximity_event +194:58|argument_list|) +194:59|argument_list|, +195:21|name|NULL +195:25|argument_list|) +195:26|expr_stmt|; +196:3|name|g_signal_connect +196:20|argument_list|( +196:21|operator|( +196:22|name|gpointer +196:30|operator|) +196:32|name|canvas +196:38|argument_list|, +196:40|literal|"proximity_out_event" +196:61|argument_list|, +197:21|name|G_CALLBACK +197:32|argument_list|( +197:33|name|on_canvas_proximity_event +197:58|argument_list|) +197:59|argument_list|, +198:21|name|NULL +198:25|argument_list|) +198:26|expr_stmt|; +199:3|name|g_signal_connect +199:20|argument_list|( +199:21|operator|( +199:22|name|gpointer +199:30|operator|) +199:32|name|canvas +199:38|argument_list|, +199:40|literal|"expose_event" +199:54|argument_list|, +200:21|name|G_CALLBACK +200:32|argument_list|( +200:33|name|on_canvas_expose_event +200:55|argument_list|) +200:56|argument_list|, +201:21|name|NULL +201:25|argument_list|) +201:26|expr_stmt|; +202:3|name|g_signal_connect +202:20|argument_list|( +202:21|operator|( +202:22|name|gpointer +202:30|operator|) +202:32|name|canvas +202:38|argument_list|, +202:40|literal|"key_press_event" +202:57|argument_list|, +203:21|name|G_CALLBACK +203:32|argument_list|( +203:33|name|on_canvas_key_press_event +203:58|argument_list|) +203:59|argument_list|, +204:21|name|NULL +204:25|argument_list|) +204:26|expr_stmt|; +205:3|name|g_signal_connect +205:20|argument_list|( +205:21|operator|( +205:22|name|gpointer +205:30|operator|) +205:32|name|canvas +205:38|argument_list|, +205:40|literal|"motion_notify_event" +205:61|argument_list|, +206:21|name|G_CALLBACK +206:32|argument_list|( +206:33|name|on_canvas_motion_notify_event +206:62|argument_list|) +206:63|argument_list|, +207:21|name|NULL +207:25|argument_list|) +207:26|expr_stmt|; +208:3|name|g_signal_connect +208:20|argument_list|( +208:21|operator|( +208:22|name|gpointer +208:30|operator|) +208:32|name|gtk_layout_get_vadjustment +208:58|argument_list|( +208:59|name|GTK_LAYOUT +208:69|argument_list|( +208:70|name|canvas +208:76|argument_list|) +208:77|argument_list|) +208:78|argument_list|, +209:21|literal|"value-changed" +209:36|argument_list|, +209:38|name|G_CALLBACK +209:49|argument_list|( +209:50|name|on_vscroll_changed +209:68|argument_list|) +209:69|argument_list|, +210:21|name|NULL +210:25|argument_list|) +210:26|expr_stmt|; +211:3|name|g_signal_connect +211:20|argument_list|( +211:21|operator|( +211:22|name|gpointer +211:30|operator|) +211:32|name|gtk_layout_get_hadjustment +211:58|argument_list|( +211:59|name|GTK_LAYOUT +211:69|argument_list|( +211:70|name|canvas +211:76|argument_list|) +211:77|argument_list|) +211:78|argument_list|, +212:21|literal|"value-changed" +212:36|argument_list|, +212:38|name|G_CALLBACK +212:49|argument_list|( +212:50|name|on_hscroll_changed +212:68|argument_list|) +212:69|argument_list|, +213:21|name|NULL +213:25|argument_list|) +213:26|expr_stmt|; +214:3|name|g_object_set_data +214:21|argument_list|( +214:22|name|G_OBJECT +214:31|argument_list|( +214:32|name|winMain +214:39|argument_list|) +214:40|argument_list|, +214:42|literal|"canvas" +214:50|argument_list|, +214:52|name|canvas +214:58|argument_list|) +214:59|expr_stmt|; +216:3|name|screen +216:10|operator|= +216:12|name|gtk_widget_get_screen +216:33|argument_list|( +216:34|name|winMain +216:41|argument_list|) +216:42|expr_stmt|; +217:3|name|ui +217:5|operator|. +217:6|name|screen_width +217:19|operator|= +217:21|name|gdk_screen_get_width +217:41|argument_list|( +217:42|name|screen +217:48|argument_list|) +217:49|expr_stmt|; +218:3|name|ui +218:5|operator|. +218:6|name|screen_height +218:20|operator|= +218:22|name|gdk_screen_get_height +218:43|argument_list|( +218:44|name|screen +218:50|argument_list|) +218:51|expr_stmt|; +220:3|name|can_xinput +220:14|operator|= +220:16|name|FALSE +220:21|expr_stmt|; +221:3|name|dev_list +221:12|operator|= +221:14|name|gdk_devices_list +221:30|argument_list|() +221:32|expr_stmt|; +222:3|while|while +222:9|condition|( +222:10|name|dev_list +222:19|operator|!= +222:22|name|NULL +222:26|condition|) +222:28|block|{ +223:5|name|device +223:12|operator|= +223:14|operator|( +223:15|name|GdkDevice +223:25|operator|* +223:26|operator|) +223:27|name|dev_list +223:35|operator|-> +223:37|name|data +223:41|expr_stmt|; +224:5|if|if +224:8|condition|( +224:9|name|device +224:16|operator|!= +224:19|name|gdk_device_get_core_pointer +224:46|argument_list|() +224:49|operator|&& +224:52|name|device +224:58|operator|-> +224:60|name|num_axes +224:69|operator|>= +224:72|literal|2 +224:73|condition|) +224:75|block|{ +225:7|comment|/* get around a GDK bug: map the valuator range CORRECTLY to [0,1] */ +226:1|ifdef|# +226:2|directive|ifdef +226:8|name|ENABLE_XINPUT_BUGFIX +227:7|name|gdk_device_set_axis_use +227:30|argument_list|( +227:31|name|device +227:37|argument_list|, +227:39|literal|0 +227:40|argument_list|, +227:42|name|GDK_AXIS_IGNORE +227:57|argument_list|) +227:58|expr_stmt|; +228:7|name|gdk_device_set_axis_use +228:30|argument_list|( +228:31|name|device +228:37|argument_list|, +228:39|literal|1 +228:40|argument_list|, +228:42|name|GDK_AXIS_IGNORE +228:57|argument_list|) +228:58|expr_stmt|; +229:1|endif|# +229:2|directive|endif +230:7|name|gdk_device_set_mode +230:26|argument_list|( +230:27|name|device +230:33|argument_list|, +230:35|name|GDK_MODE_SCREEN +230:50|argument_list|) +230:51|expr_stmt|; +231:7|if|if +231:10|condition|( +231:11|name|g_strrstr +231:20|argument_list|( +231:21|name|device +231:27|operator|-> +231:29|name|name +231:33|argument_list|, +231:35|literal|"raser" +231:42|argument_list|) +231:43|condition|) +232:9|name|gdk_device_set_source +232:30|argument_list|( +232:31|name|device +232:37|argument_list|, +232:39|name|GDK_SOURCE_ERASER +232:56|argument_list|) +232:57|expr_stmt|; +233:7|name|can_xinput +233:18|operator|= +233:20|name|TRUE +233:24|expr_stmt|; +234:5|block|} +235:5|name|dev_list +235:14|operator|= +235:16|name|dev_list +235:24|operator|-> +235:26|name|next +235:30|expr_stmt|; +236:3|block|} +237:3|if|if +237:6|condition|( +237:7|operator|! +237:8|name|can_xinput +237:18|condition|) +238:5|name|gtk_widget_set_sensitive +238:29|argument_list|( +238:30|name|GET_COMPONENT +238:43|argument_list|( +238:44|literal|"optionsUseXInput" +238:62|argument_list|) +238:63|argument_list|, +238:65|name|FALSE +238:70|argument_list|) +238:71|expr_stmt|; +240:3|name|ui +240:5|operator|. +240:6|name|use_xinput +240:17|operator|= +240:19|name|ui +240:21|operator|. +240:22|name|allow_xinput +240:35|operator|&& +240:38|name|can_xinput +240:48|expr_stmt|; +242:3|name|gtk_check_menu_item_set_active +242:33|argument_list|( +243:5|name|GTK_CHECK_MENU_ITEM +243:24|argument_list|( +243:25|name|GET_COMPONENT +243:38|argument_list|( +243:39|literal|"optionsProgressiveBG" +243:61|argument_list|) +243:62|argument_list|) +243:63|argument_list|, +243:65|name|ui +243:67|operator|. +243:68|name|progressive_bg +243:82|argument_list|) +243:83|expr_stmt|; +244:3|name|gtk_check_menu_item_set_active +244:33|argument_list|( +245:5|name|GTK_CHECK_MENU_ITEM +245:24|argument_list|( +245:25|name|GET_COMPONENT +245:38|argument_list|( +245:39|literal|"optionsPrintRuling" +245:59|argument_list|) +245:60|argument_list|) +245:61|argument_list|, +245:63|name|ui +245:65|operator|. +245:66|name|print_ruling +245:78|argument_list|) +245:79|expr_stmt|; +246:3|name|gtk_check_menu_item_set_active +246:33|argument_list|( +247:5|name|GTK_CHECK_MENU_ITEM +247:24|argument_list|( +247:25|name|GET_COMPONENT +247:38|argument_list|( +247:39|literal|"optionsLegacyPDFExport" +247:63|argument_list|) +247:64|argument_list|) +247:65|argument_list|, +247:67|name|ui +247:69|operator|. +247:70|name|exportpdf_prefer_legacy +247:93|argument_list|) +247:94|expr_stmt|; +248:3|name|gtk_check_menu_item_set_active +248:33|argument_list|( +249:5|name|GTK_CHECK_MENU_ITEM +249:24|argument_list|( +249:25|name|GET_COMPONENT +249:38|argument_list|( +249:39|literal|"optionsLayersPDFExport" +249:63|argument_list|) +249:64|argument_list|) +249:65|argument_list|, +249:67|name|ui +249:69|operator|. +249:70|name|exportpdf_layers +249:86|argument_list|) +249:87|expr_stmt|; +250:3|name|gtk_check_menu_item_set_active +250:33|argument_list|( +251:5|name|GTK_CHECK_MENU_ITEM +251:24|argument_list|( +251:25|name|GET_COMPONENT +251:38|argument_list|( +251:39|literal|"optionsAutoloadPdfXoj" +251:62|argument_list|) +251:63|argument_list|) +251:64|argument_list|, +251:66|name|ui +251:68|operator|. +251:69|name|autoload_pdf_xoj +251:85|argument_list|) +251:86|expr_stmt|; +252:3|name|gtk_check_menu_item_set_active +252:33|argument_list|( +253:5|name|GTK_CHECK_MENU_ITEM +253:24|argument_list|( +253:25|name|GET_COMPONENT +253:38|argument_list|( +253:39|literal|"optionsAutosaveXoj" +253:59|argument_list|) +253:60|argument_list|) +253:61|argument_list|, +253:63|name|ui +253:65|operator|. +253:66|name|autosave_enabled +253:82|argument_list|) +253:83|expr_stmt|; +254:3|name|gtk_check_menu_item_set_active +254:33|argument_list|( +255:5|name|GTK_CHECK_MENU_ITEM +255:24|argument_list|( +255:25|name|GET_COMPONENT +255:38|argument_list|( +255:39|literal|"optionsLeftHanded" +255:58|argument_list|) +255:59|argument_list|) +255:60|argument_list|, +255:62|name|ui +255:64|operator|. +255:65|name|left_handed +255:76|argument_list|) +255:77|expr_stmt|; +256:3|name|gtk_check_menu_item_set_active +256:33|argument_list|( +257:5|name|GTK_CHECK_MENU_ITEM +257:24|argument_list|( +257:25|name|GET_COMPONENT +257:38|argument_list|( +257:39|literal|"optionsShortenMenus" +257:60|argument_list|) +257:61|argument_list|) +257:62|argument_list|, +257:64|name|ui +257:66|operator|. +257:67|name|shorten_menus +257:80|argument_list|) +257:81|expr_stmt|; +258:3|name|gtk_check_menu_item_set_active +258:33|argument_list|( +259:5|name|GTK_CHECK_MENU_ITEM +259:24|argument_list|( +259:25|name|GET_COMPONENT +259:38|argument_list|( +259:39|literal|"optionsAutoSavePrefs" +259:61|argument_list|) +259:62|argument_list|) +259:63|argument_list|, +259:65|name|ui +259:67|operator|. +259:68|name|auto_save_prefs +259:83|argument_list|) +259:84|expr_stmt|; +260:3|name|gtk_check_menu_item_set_active +260:33|argument_list|( +261:5|name|GTK_CHECK_MENU_ITEM +261:24|argument_list|( +261:25|name|GET_COMPONENT +261:38|argument_list|( +261:39|literal|"optionsButtonSwitchMapping" +261:67|argument_list|) +261:68|argument_list|) +261:69|argument_list|, +261:71|name|ui +261:73|operator|. +261:74|name|button_switch_mapping +261:95|argument_list|) +261:96|expr_stmt|; +262:3|name|gtk_check_menu_item_set_active +262:33|argument_list|( +263:5|name|GTK_CHECK_MENU_ITEM +263:24|argument_list|( +263:25|name|GET_COMPONENT +263:38|argument_list|( +263:39|literal|"optionsPenCursor" +263:57|argument_list|) +263:58|argument_list|) +263:59|argument_list|, +263:61|name|ui +263:63|operator|. +263:64|name|pen_cursor +263:74|argument_list|) +263:75|expr_stmt|; +265:3|name|hide_unimplemented +265:21|argument_list|() +265:23|expr_stmt|; +267:3|name|update_undo_redo_enabled +267:27|argument_list|() +267:29|expr_stmt|; +268:3|name|update_copy_paste_enabled +268:28|argument_list|() +268:30|expr_stmt|; +269:3|name|update_vbox_order +269:20|argument_list|( +269:21|name|ui +269:23|operator|. +269:24|name|vertical_order +269:38|index|[ +269:39|name|ui +269:41|operator|. +269:42|name|fullscreen +269:52|condition|? +269:53|literal|1 +269:54|else|: +269:55|literal|0 +269:56|index|] +269:57|argument_list|) +269:58|expr_stmt|; +270:3|name|gtk_widget_grab_focus +270:24|argument_list|( +270:25|name|GTK_WIDGET +270:35|argument_list|( +270:36|name|canvas +270:42|argument_list|) +270:43|argument_list|) +270:44|expr_stmt|; +272:3|comment|// show everything... +274:3|name|gtk_widget_show +274:19|argument_list|( +274:20|name|winMain +274:27|argument_list|) +274:28|expr_stmt|; +275:3|name|update_cursor +275:16|argument_list|() +275:18|expr_stmt|; +277:3|comment|/* this will cause extension events to get enabled/disabled, but we need the windows to be mapped first */ +279:3|name|gtk_check_menu_item_set_active +279:33|argument_list|( +280:5|name|GTK_CHECK_MENU_ITEM +280:24|argument_list|( +280:25|name|GET_COMPONENT +280:38|argument_list|( +280:39|literal|"optionsUseXInput" +280:57|argument_list|) +280:58|argument_list|) +280:59|argument_list|, +280:61|name|ui +280:63|operator|. +280:64|name|use_xinput +280:74|argument_list|) +280:75|expr_stmt|; +282:3|comment|/* fix a bug in GTK+ 2.16 and 2.17: scrollbars shouldn't get extended input events from pointer motion when cursor moves into main window */ +285:3|if|if +285:6|condition|( +285:7|operator|! +285:8|name|gtk_check_version +285:25|argument_list|( +285:26|literal|2 +285:27|argument_list|, +285:29|literal|16 +285:31|argument_list|, +285:33|literal|0 +285:34|argument_list|) +285:35|condition|) +285:37|block|{ +286:5|name|g_signal_connect +286:22|argument_list|( +287:7|name|GET_COMPONENT +287:20|argument_list|( +287:21|literal|"menubar" +287:30|argument_list|) +287:31|argument_list|, +288:7|literal|"event" +288:14|argument_list|, +288:16|name|G_CALLBACK +288:27|argument_list|( +288:28|name|filter_extended_events +288:50|argument_list|) +288:51|argument_list|, +289:7|name|NULL +289:11|argument_list|) +289:12|expr_stmt|; +290:5|name|g_signal_connect +290:22|argument_list|( +291:7|name|GET_COMPONENT +291:20|argument_list|( +291:21|literal|"toolbarMain" +291:34|argument_list|) +291:35|argument_list|, +292:7|literal|"event" +292:14|argument_list|, +292:16|name|G_CALLBACK +292:27|argument_list|( +292:28|name|filter_extended_events +292:50|argument_list|) +292:51|argument_list|, +293:7|name|NULL +293:11|argument_list|) +293:12|expr_stmt|; +294:5|name|g_signal_connect +294:22|argument_list|( +295:7|name|GET_COMPONENT +295:20|argument_list|( +295:21|literal|"toolbarPen" +295:33|argument_list|) +295:34|argument_list|, +296:7|literal|"event" +296:14|argument_list|, +296:16|name|G_CALLBACK +296:27|argument_list|( +296:28|name|filter_extended_events +296:50|argument_list|) +296:51|argument_list|, +297:7|name|NULL +297:11|argument_list|) +297:12|expr_stmt|; +298:5|name|g_signal_connect +298:22|argument_list|( +299:7|name|GET_COMPONENT +299:20|argument_list|( +299:21|literal|"statusbar" +299:32|argument_list|) +299:33|argument_list|, +300:7|literal|"event" +300:14|argument_list|, +300:16|name|G_CALLBACK +300:27|argument_list|( +300:28|name|filter_extended_events +300:50|argument_list|) +300:51|argument_list|, +301:7|name|NULL +301:11|argument_list|) +301:12|expr_stmt|; +302:5|name|g_signal_connect +302:22|argument_list|( +303:7|operator|( +303:8|name|gpointer +303:16|operator|) +303:17|operator|( +303:18|name|gtk_scrolled_window_get_vscrollbar +303:52|argument_list|( +303:53|name|GTK_SCROLLED_WINDOW +303:72|argument_list|( +303:73|name|w +303:74|argument_list|) +303:75|argument_list|) +303:76|operator|) +303:77|argument_list|, +304:7|literal|"event" +304:14|argument_list|, +304:16|name|G_CALLBACK +304:27|argument_list|( +304:28|name|filter_extended_events +304:50|argument_list|) +304:51|argument_list|, +305:7|name|NULL +305:11|argument_list|) +305:12|expr_stmt|; +306:5|name|g_signal_connect +306:22|argument_list|( +307:7|operator|( +307:8|name|gpointer +307:16|operator|) +307:17|operator|( +307:18|name|gtk_scrolled_window_get_hscrollbar +307:52|argument_list|( +307:53|name|GTK_SCROLLED_WINDOW +307:72|argument_list|( +307:73|name|w +307:74|argument_list|) +307:75|argument_list|) +307:76|operator|) +307:77|argument_list|, +308:7|literal|"event" +308:14|argument_list|, +308:16|name|G_CALLBACK +308:27|argument_list|( +308:28|name|filter_extended_events +308:50|argument_list|) +308:51|argument_list|, +309:7|name|NULL +309:11|argument_list|) +309:12|expr_stmt|; +310:3|block|} +312:3|comment|// load the MRU +314:3|name|init_mru +314:11|argument_list|() +314:13|expr_stmt|; +316:3|comment|// and finally, open a file specified on the command line +317:3|comment|// (moved here because display parameters weren't initialized yet...) +319:3|if|if +319:6|condition|( +319:7|name|argc +319:12|operator|== +319:15|literal|1 +319:16|condition|) +319:18|return|return; +320:3|name|set_cursor_busy +320:18|argument_list|( +320:19|name|TRUE +320:23|argument_list|) +320:24|expr_stmt|; +321:3|if|if +321:6|condition|( +321:7|name|g_path_is_absolute +321:25|argument_list|( +321:26|name|argv +321:30|index|[ +321:31|literal|1 +321:32|index|] +321:33|argument_list|) +321:34|condition|) +322:5|name|tmpfn +322:11|operator|= +322:13|name|g_strdup +322:21|argument_list|( +322:22|name|argv +322:26|index|[ +322:27|literal|1 +322:28|index|] +322:29|argument_list|) +322:30|expr_stmt|; +323:3|else|else +323:8|block|{ +324:5|name|tmppath +324:13|operator|= +324:15|name|g_get_current_dir +324:32|argument_list|() +324:34|expr_stmt|; +325:5|name|tmpfn +325:11|operator|= +325:13|name|g_build_filename +325:29|argument_list|( +325:30|name|tmppath +325:37|argument_list|, +325:39|name|argv +325:43|index|[ +325:44|literal|1 +325:45|index|] +325:46|argument_list|, +325:48|name|NULL +325:52|argument_list|) +325:53|expr_stmt|; +326:5|name|g_free +326:11|argument_list|( +326:12|name|tmppath +326:19|argument_list|) +326:20|expr_stmt|; +327:3|block|} +328:3|name|success +328:11|operator|= +328:13|name|open_journal +328:25|argument_list|( +328:26|name|tmpfn +328:31|argument_list|) +328:32|expr_stmt|; +329:3|name|g_free +329:9|argument_list|( +329:10|name|tmpfn +329:15|argument_list|) +329:16|expr_stmt|; +330:3|name|set_cursor_busy +330:18|argument_list|( +330:19|name|FALSE +330:24|argument_list|) +330:25|expr_stmt|; +331:3|if|if +331:6|condition|( +331:7|operator|! +331:8|name|success +331:15|condition|) +331:17|block|{ +332:5|name|w +332:7|operator|= +332:9|name|gtk_message_dialog_new +332:31|argument_list|( +332:32|name|GTK_WINDOW +332:43|argument_list|( +332:44|name|winMain +332:51|argument_list|) +332:52|argument_list|, +332:54|name|GTK_DIALOG_DESTROY_WITH_PARENT +332:84|argument_list|, +333:8|name|GTK_MESSAGE_ERROR +333:25|argument_list|, +333:27|name|GTK_BUTTONS_OK +333:41|argument_list|, +333:43|name|_ +333:44|argument_list|( +333:45|literal|"Error opening file '%s'" +333:70|argument_list|) +333:71|argument_list|, +333:73|name|argv +333:77|index|[ +333:78|literal|1 +333:79|index|] +333:80|argument_list|) +333:81|expr_stmt|; +334:5|name|wrapper_gtk_dialog_run +334:27|argument_list|( +334:28|name|GTK_DIALOG +334:38|argument_list|( +334:39|name|w +334:40|argument_list|) +334:41|argument_list|) +334:42|expr_stmt|; +335:5|name|gtk_widget_destroy +335:23|argument_list|( +335:24|name|w +335:25|argument_list|) +335:26|expr_stmt|; +336:3|block|} +337:1|block|} +-:-|end_function +-:-| +-:-|begin_function +340:1|name|int +341:-|DECL|function|main (int argc,char * argv[]) +341:1|name|main +341:6|parameter_list|( +341:7|name|int +341:11|name|argc +341:15|parameter_list|, +341:17|name|char +341:22|modifier|* +341:23|name|argv +341:27|index|[] +341:29|parameter_list|) +342:1|block|{ +343:3|name|gchar +343:9|modifier|* +343:10|name|path +343:14|decl_stmt|, +343:16|modifier|* +343:17|name|path1 +343:22|decl_stmt|, +343:24|modifier|* +343:25|name|path2 +343:30|decl_stmt|; +345:1|ifdef|# +345:2|directive|ifdef +345:8|name|ENABLE_NLS +346:3|name|bindtextdomain +346:18|argument_list|( +346:19|name|GETTEXT_PACKAGE +346:34|argument_list|, +346:36|name|PACKAGE_LOCALE_DIR +346:54|argument_list|) +346:55|expr_stmt|; +347:3|name|bind_textdomain_codeset +347:27|argument_list|( +347:28|name|GETTEXT_PACKAGE +347:43|argument_list|, +347:45|literal|"UTF-8" +347:52|argument_list|) +347:53|expr_stmt|; +348:3|name|textdomain +348:14|argument_list|( +348:15|name|GETTEXT_PACKAGE +348:30|argument_list|) +348:31|expr_stmt|; +349:1|endif|# +349:2|directive|endif +351:3|name|gtk_set_locale +351:18|argument_list|() +351:20|expr_stmt|; +352:3|name|gtk_init +352:12|argument_list|( +352:13|operator|& +352:14|name|argc +352:18|argument_list|, +352:20|operator|& +352:21|name|argv +352:25|argument_list|) +352:26|expr_stmt|; +354:3|name|path +354:8|operator|= +354:10|name|g_path_get_dirname +354:28|argument_list|( +354:29|name|argv +354:33|index|[ +354:34|literal|0 +354:35|index|] +354:36|argument_list|) +354:37|expr_stmt|; +355:3|name|path1 +355:9|operator|= +355:11|name|g_build_filename +355:27|argument_list|( +355:28|name|path +355:32|argument_list|, +355:34|literal|"pixmaps" +355:43|argument_list|, +355:45|name|NULL +355:49|argument_list|) +355:50|expr_stmt|; +356:3|name|path2 +356:9|operator|= +356:11|name|g_build_filename +356:27|argument_list|( +356:28|name|path +356:32|argument_list|, +356:34|literal|".." +356:38|argument_list|, +356:40|literal|"pixmaps" +356:49|argument_list|, +356:51|name|NULL +356:55|argument_list|) +356:56|expr_stmt|; +357:3|name|add_pixmap_directory +357:24|argument_list|( +357:25|name|path +357:29|argument_list|) +357:30|expr_stmt|; +358:3|name|add_pixmap_directory +358:24|argument_list|( +358:25|name|path2 +358:30|argument_list|) +358:31|expr_stmt|; +359:3|name|add_pixmap_directory +359:24|argument_list|( +359:25|name|path1 +359:30|argument_list|) +359:31|expr_stmt|; +360:3|name|g_free +360:9|argument_list|( +360:10|name|path +360:14|argument_list|) +360:15|expr_stmt|; +361:3|name|g_free +361:9|argument_list|( +361:10|name|path1 +361:15|argument_list|) +361:16|expr_stmt|; +362:3|name|g_free +362:9|argument_list|( +362:10|name|path2 +362:15|argument_list|) +362:16|expr_stmt|; +363:3|name|add_pixmap_directory +363:24|argument_list|( +363:25|name|PACKAGE_DATA_DIR +363:42|literal|"/" +363:46|name|PACKAGE +363:54|literal|"/pixmaps" +363:64|argument_list|) +363:65|expr_stmt|; +365:3|comment|/* * The following code was added by Glade to create one of each component * (except popup menus), just so that you see something after building * the project. Delete any components that you don't want shown initially. */ +370:3|name|winMain +370:11|operator|= +370:13|name|create_winMain +370:28|argument_list|() +370:30|expr_stmt|; +372:3|name|init_stuff +372:14|argument_list|( +372:15|name|argc +372:19|argument_list|, +372:21|name|argv +372:25|argument_list|) +372:26|expr_stmt|; +373:3|name|gtk_window_set_icon +373:22|argument_list|( +373:23|name|GTK_WINDOW +373:33|argument_list|( +373:34|name|winMain +373:41|argument_list|) +373:42|argument_list|, +373:44|name|create_pixbuf +373:57|argument_list|( +373:58|literal|"xournal.png" +373:71|argument_list|) +373:72|argument_list|) +373:73|expr_stmt|; +375:3|name|gtk_main +375:12|argument_list|() +375:14|expr_stmt|; +377:3|if|if +377:6|condition|( +377:7|name|bgpdf +377:12|operator|. +377:13|name|status +377:20|operator|!= +377:23|name|STATUS_NOT_INIT +377:38|condition|) +377:40|name|shutdown_bgpdf +377:54|argument_list|() +377:56|expr_stmt|; +379:3|name|save_mru_list +379:16|argument_list|() +379:18|expr_stmt|; +380:3|name|autosave_cleanup +380:19|argument_list|( +380:20|operator|& +380:21|name|ui +380:23|operator|. +380:24|name|autosave_filename_list +380:46|argument_list|) +380:47|expr_stmt|; +381:3|if|if +381:6|condition|( +381:7|name|ui +381:9|operator|. +381:10|name|auto_save_prefs +381:25|condition|) +381:27|name|save_config_to_file +381:46|argument_list|() +381:48|expr_stmt|; +383:3|return|return +383:10|literal|0 +383:11|return|; +384:1|block|} +-:-|end_function +-:-| +-:-|end_unit +-:-| diff --git a/tokenize/t/tokenize.t b/tokenize/t/tokenize.t new file mode 100644 index 00000000..6fe1371d --- /dev/null +++ b/tokenize/t/tokenize.t @@ -0,0 +1,67 @@ +#!/usr/bin/env perl + +# Tests for the tokenize.pl language dispatcher. It forwards to +# tokenizeSrcMl.pl (run with its built-in defaults, which is also how the +# dispatcher runs in production), so its output must be byte-identical to +# calling tokenizeSrcMl.pl directly with the same flags. + +use strict; +use warnings; +use Test::More; +use FindBin; +use File::Temp qw(tempdir); + +my $dispatcher = "$FindBin::Bin/../tokenize.pl"; +my $direct = "$FindBin::Bin/../tokenizeSrcMl.pl"; +my $fixtures = "$FindBin::Bin/../srcMLtoken/tests"; + +plan skip_all => "srcml not on PATH" + unless system("srcml --version >/dev/null 2>&1") == 0; +plan skip_all => "srcml2token not built (cd tokenize/srcMLtoken && make)" + unless -x "$FindBin::Bin/../srcMLtoken/srcml2token"; + +plan tests => 7; + +my $workdir = tempdir(CLEANUP => 1); + +sub slurp { + my ($file) = @_; + open(my $fh, '<', $file) or die "unable to read [$file]: $!"; + local $/; + my $content = <$fh>; + return defined $content ? $content : ''; +} + +# returns (exit status, stdout, stderr) +sub run_script { + my ($script, @args) = @_; + my $out = "$workdir/stdout"; + my $err = "$workdir/stderr"; + my $status = system("perl '$script' " . join(' ', @args) . " > '$out' 2> '$err'"); + return ($status, slurp($out), slurp($err)); +} + +# dispatch by explicit --language forwards flags and produces the same +# output as running tokenizeSrcMl.pl directly +{ + my ($dstatus, $dout) = run_script($dispatcher, "--language=C", "--position", "'$fixtures/main.c'"); + my ($sstatus, $sout) = run_script($direct, "--language=C", "--position", "'$fixtures/main.c'"); + is($dstatus, 0, "dispatching main.c as C succeeds"); + is($sstatus, 0, "direct tokenizeSrcMl.pl run succeeds"); + is($dout, $sout, "dispatcher output is byte-identical to the direct run"); +} + +# language autodetection from the file extension picks the same parser +{ + my ($astatus, $aout) = run_script($dispatcher, "--position", "'$fixtures/main.c'"); + my ($estatus, $eout) = run_script($dispatcher, "--language=C", "--position", "'$fixtures/main.c'"); + is($astatus, 0, "autodetecting .c succeeds"); + is($aout, $eout, "autodetected output matches the explicit --language=C run"); +} + +# unknown --language is rejected +{ + my ($status, $out, $err) = run_script($dispatcher, "--language=Nope", "'$fixtures/main.c'"); + isnt($status, 0, "unknown --language exits non-zero"); + like($err, qr/We do not know what to do/, "unknown language reported on stderr"); +} diff --git a/tokenize/t/tokenizeSrcMl.t b/tokenize/t/tokenizeSrcMl.t new file mode 100644 index 00000000..3ecf750f --- /dev/null +++ b/tokenize/t/tokenizeSrcMl.t @@ -0,0 +1,95 @@ +#!/usr/bin/env perl + +# Golden-file tests for tokenizeSrcMl.pl, mirroring the srcMLtoken pattern: +# run the tokenizer over committed fixtures and compare against t/expected/. +# +# Requires srcml and ctags on PATH and tokenize/srcMLtoken/srcml2token built +# (cd tokenize/srcMLtoken && make); skipped otherwise. +# +# To regenerate the goldens after an intentional output change, run inside +# the devenv shell, from the repo root: +# perl tokenize/tokenizeSrcMl.pl --ctags=ctags \ +# --srcml2token=$PWD/tokenize/srcMLtoken/srcml2token \ +# --position tokenize/srcMLtoken/tests/main.c tokenize/t/expected/main.c.token +# perl tokenize/tokenizeSrcMl.pl --ctags=ctags \ +# --srcml2token=$PWD/tokenize/srcMLtoken/srcml2token \ +# tokenize/srcMLtoken/tests/main.c tokenize/t/expected/main.c.nopos.token +# perl tokenize/tokenizeSrcMl.pl --ctags=ctags \ +# --srcml2token=$PWD/tokenize/srcMLtoken/srcml2token \ +# --position tokenize/srcMLtoken/tests/StringUtil.java tokenize/t/expected/StringUtil.java.token + +use strict; +use warnings; +use Test::More; +use FindBin; +use File::Temp qw(tempdir); + +my $script = "$FindBin::Bin/../tokenizeSrcMl.pl"; +my $srcml2token = "$FindBin::Bin/../srcMLtoken/srcml2token"; +my $fixtures = "$FindBin::Bin/../srcMLtoken/tests"; +my $expected = "$FindBin::Bin/expected"; + +plan skip_all => "srcml not on PATH" + unless system("srcml --version >/dev/null 2>&1") == 0; +plan skip_all => "ctags not on PATH" + unless system("ctags --version >/dev/null 2>&1") == 0; +plan skip_all => "srcml2token not built (cd tokenize/srcMLtoken && make)" + unless -x $srcml2token; + +plan tests => 8; + +my $workdir = tempdir(CLEANUP => 1); + +sub slurp { + my ($file) = @_; + open(my $fh, '<', $file) or die "unable to read [$file]: $!"; + local $/; + my $content = <$fh>; + return defined $content ? $content : ''; +} + +# returns (exit status, stdout, stderr) +sub run_tokenizer { + my (@args) = @_; + my $out = "$workdir/stdout"; + my $err = "$workdir/stderr"; + my $status = system("perl '$script' --ctags=ctags --srcml2token='$srcml2token' " + . join(' ', @args) . " > '$out' 2> '$err'"); + return ($status, slurp($out), slurp($err)); +} + +# C fixture, with token positions +{ + my ($status, $out, $err) = run_tokenizer("--position", "'$fixtures/main.c'"); + is($status, 0, "tokenizing main.c with --position succeeds"); + is($out, slurp("$expected/main.c.token"), + "main.c --position output matches the golden file"); +} + +# C fixture, without positions +{ + my ($status, $out, $err) = run_tokenizer("'$fixtures/main.c'"); + is($status, 0, "tokenizing main.c without --position succeeds"); + is($out, slurp("$expected/main.c.nopos.token"), + "main.c output matches the golden file"); +} + +# Java fixture (language autodetected from the .java extension) +{ + my ($status, $out, $err) = run_tokenizer("--position", "'$fixtures/StringUtil.java'"); + is($status, 0, "tokenizing StringUtil.java succeeds"); + is($out, slurp("$expected/StringUtil.java.token"), + "StringUtil.java output matches the golden file"); +} + +# unknown extension without --language is an error +{ + my $bogus = "$workdir/mystery.xyz"; + open(my $fh, '>', $bogus) or die $!; + print $fh "int main() {}\n"; + close $fh; + + my ($status, $out, $err) = run_tokenizer("'$bogus'"); + isnt($status, 0, "unknown extension exits non-zero"); + like($err, qr/Unknown extension/, "unknown extension reported on stderr"); +} diff --git a/tokenizeByBlobId/t/tokenBySha.t b/tokenizeByBlobId/t/tokenBySha.t new file mode 100644 index 00000000..769b29c7 --- /dev/null +++ b/tokenizeByBlobId/t/tokenBySha.t @@ -0,0 +1,138 @@ +#!/usr/bin/env perl + +# Tests for tokenBySha.pl, the per-blob driver invoked by blobExec. It is +# argv-less by design: the blob arrives on stdin and everything else comes +# from BFG_* environment variables. A stub tokenizer stands in for the real +# tokenize command, so neither srcml nor ctags is needed. +# +# Note: the script anchors its scratch files to tokenizeByBlobId/build/ (it +# cleans them up itself); leftovers there from a failed run are scratch, not +# fixtures. + +use strict; +use warnings; +use Test::More tests => 13; +use FindBin; +use File::Temp qw(tempdir); +use Digest::SHA qw(sha1_hex); + +my $script = "$FindBin::Bin/../tokenBySha.pl"; +my $workdir = tempdir(CLEANUP => 1); + +# stub tokenizer: prints a banner, its --language argument, then the file +# it was given (the temp copy of the blob). Deterministic output only. +my $stub = "$workdir/stub-tokenizer.sh"; +{ + open(my $fh, '>', $stub) or die $!; + print $fh "#!/bin/sh\necho STUB-TOKENIZER\necho \"\$1\"\ncat \"\$2\"\n"; + close $fh; + chmod 0755, $stub or die $!; +} + +sub slurp { + my ($file) = @_; + open(my $fh, '<', $file) or die "unable to read [$file]: $!"; + local $/; + my $content = <$fh>; + return defined $content ? $content : ''; +} + +# runs tokenBySha.pl with the given env overrides and stdin content; +# returns (exit status, stdout, stderr) +sub run_tokenbysha { + my ($content, %env) = @_; + my $in = "$workdir/stdin"; + my $out = "$workdir/stdout"; + my $err = "$workdir/stderr"; + open(my $fh, '>', $in) or die $!; + print $fh $content; + close $fh; + + local %ENV = %ENV; + while (my ($k, $v) = each %env) { + if (defined $v) { $ENV{$k} = $v } else { delete $ENV{$k} } + } + + my $status = system("perl '$script' < '$in' > '$out' 2> '$err'"); + return ($status, slurp($out), slurp($err)); +} + +my $memoDir = tempdir(CLEANUP => 1); +my $content = "int a;\nint b;\n"; +my $sha1 = sha1_hex($content); +my $memoFile = "$memoDir/" . substr($sha1, 0, 2) . "/" . substr($sha1, 2, 2) . "/$sha1"; + +# first run: tokenizes via the stub and memoizes +{ + my ($status, $out, $err) = run_tokenbysha($content, + BFG_MEMO_DIR => $memoDir, + BFG_TOKENIZE_CMD => $stub, + BFG_BLOB => "0" x 40, + BFG_FILENAME => "foo.c", + ); + is($status, 0, "first run succeeds"); + like($out, qr/^STUB-TOKENIZER\n--language=C\n/, + "stub is invoked with --language=C for a .c file"); + is($out, "STUB-TOKENIZER\n--language=C\n$content", + "blob content reaches the tokenizer"); + ok(-f $memoFile, "output is memoized under xx/yy/ of the blob content"); + is(slurp($memoFile), $out, "memoized file matches stdout"); +} + +# second run: served from the memo without invoking the tokenize command +{ + my ($status, $out, $err) = run_tokenbysha($content, + BFG_MEMO_DIR => $memoDir, + BFG_TOKENIZE_CMD => "/bin/false", + BFG_BLOB => "0" x 40, + BFG_FILENAME => "foo.c", + ); + is($status, 0, "cache hit succeeds even with a broken tokenize command"); + is($out, slurp($memoFile), "cache hit replays the memoized output"); +} + +# extension mapping: .cpp maps to C++ +{ + my ($status, $out, $err) = run_tokenbysha("class X {};\n", + BFG_MEMO_DIR => $memoDir, + BFG_TOKENIZE_CMD => $stub, + BFG_BLOB => "1" x 40, + BFG_FILENAME => "foo.cpp", + ); + is($status, 0, ".cpp run succeeds"); + like($out, qr/^STUB-TOKENIZER\n--language=C\+\+\n/, ".cpp maps to --language=C++"); +} + +# unknown extension dies +{ + my ($status, $out, $err) = run_tokenbysha($content, + BFG_MEMO_DIR => $memoDir, + BFG_TOKENIZE_CMD => $stub, + BFG_BLOB => "2" x 40, + BFG_FILENAME => "foo.zzz", + ); + isnt($status, 0, "unknown extension exits non-zero"); + like($err, qr/unknown file extension/, "unknown extension reported on stderr"); +} + +# missing memo dir dies before doing any work +{ + my ($status, $out, $err) = run_tokenbysha($content, + BFG_MEMO_DIR => undef, + BFG_TOKENIZE_CMD => $stub, + BFG_BLOB => "3" x 40, + BFG_FILENAME => "foo.c", + ); + isnt($status, 0, "missing BFG_MEMO_DIR exits non-zero"); +} + +# empty BFG_FILENAME dies +{ + my ($status, $out, $err) = run_tokenbysha($content, + BFG_MEMO_DIR => $memoDir, + BFG_TOKENIZE_CMD => $stub, + BFG_BLOB => "4" x 40, + BFG_FILENAME => "", + ); + isnt($status, 0, "empty BFG_FILENAME exits non-zero"); +} From 1256e54b1b00fcd7fb971d0a42f8a4a65339bd10 Mon Sep 17 00:00:00 2001 From: Lucca Oliveira Date: Fri, 3 Jul 2026 08:58:47 -0300 Subject: [PATCH 06/12] ci: run the new Scala and Perl test suites sbt builds become 'test assembly'/'test one-jar', and a prove step runs the Perl suites after srcml2token is built (the golden tests need it). --- .github/workflows/ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b35b370..74a8f13a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,8 +56,8 @@ jobs: restore-keys: | cargo-${{ runner.os }}- - - name: Build blobExec (sbt assembly, Scala 2.13 / JDK 21) - run: devenv shell -- bash -c 'cd blobExec && sbt -batch assembly' + - name: Build & test blobExec (sbt test assembly, Scala 2.13 / JDK 21) + run: devenv shell -- bash -c 'cd blobExec && sbt -batch test assembly' - name: Build & test C++ transcoder (srcml2token) run: devenv shell -- bash -c 'cd tokenize/srcMLtoken && make && make test' @@ -65,12 +65,15 @@ jobs: - name: Build & test Rust tokenizer (rust_tokenizer) run: devenv shell -- bash -c 'cd tokenize/rustTokenizer && make && make test' - - name: Build legacy onejar modules (sbt one-jar, Scala 2.10 / JDK 8) + - name: Perl tests (prove) + run: devenv shell -- bash -c 'prove tokenize/t tokenizeByBlobId/t blameRepo/t prettyPrint/t' + + - name: Build & test legacy onejar modules (sbt test one-jar, Scala 2.10 / JDK 8) run: | devenv shell -- bash -c ' set -e for m in slickGitLog persons remapCommits; do echo "::group::$m" - ( cd "$m" && sbt --java-home "$LEGACY_JAVA_HOME" -batch one-jar ) + ( cd "$m" && sbt --java-home "$LEGACY_JAVA_HOME" -batch test one-jar ) echo "::endgroup::" done' From bb70a09c77e21f757929e211c040887653293155 Mon Sep 17 00:00:00 2001 From: Lucca Oliveira Date: Tue, 4 Aug 2026 11:32:52 -0300 Subject: [PATCH 07/12] test(scala): cover legacy module outputs end to end --- persons/src/test/scala/unifyPersonsSpec.scala | 91 +++++++++++++++++++ .../src/test/scala/remapCommitsSpec.scala | 81 +++++++++++++++++ .../src/test/scala/gitLogToDbSpec.scala | 46 ++++++++++ 3 files changed, 218 insertions(+) diff --git a/persons/src/test/scala/unifyPersonsSpec.scala b/persons/src/test/scala/unifyPersonsSpec.scala index 636252b1..0fb01edf 100644 --- a/persons/src/test/scala/unifyPersonsSpec.scala +++ b/persons/src/test/scala/unifyPersonsSpec.scala @@ -18,6 +18,14 @@ along with this program. If not, see . import org.scalatest.FunSuite import unifyPersons.Person +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.lib.PersonIdent + +import java.io.{File, PrintWriter} +import java.nio.file.Files +import java.sql.DriverManager +import java.util.{Date, TimeZone} + class unifyPersonsSpec extends FunSuite { def mkPerson(name: String, email: String): Person = { @@ -136,4 +144,87 @@ class unifyPersonsSpec extends FunSuite { val full = mkPerson("Rudy Root", "rudy@example.com") assert(unifyPersons.preferredName(List(single, full)) === "root@example.com") } + + def withTempRepo(testCode: (Git, File) => Unit): Unit = { + val dir = Files.createTempDirectory("unifyPersonsSpec").toFile + val git = Git.init.setDirectory(dir).call() + try { + testCode(git, dir) + } finally { + git.close() + def rm(file: File): Unit = { + if (file.isDirectory) file.listFiles.foreach(rm) + file.delete() + } + rm(dir) + } + } + + def commitFile( + git: Git, + dir: File, + name: String, + content: String, + who: PersonIdent, + message: String) = { + val out = new PrintWriter(new File(dir, name)) + out.print(content) + out.close() + git.add.addFilepattern(name).call() + git.commit.setAuthor(who).setCommitter(who).setMessage(message).call() + } + + test("main writes the identity spreadsheet and persons database") { + withTempRepo { (git, dir) => + val utc = TimeZone.getTimeZone("UTC") + val alice = new PersonIdent( + "Alice Coder", + "alice@example.com", + new Date(1500000000000L), + utc) + val bob = new PersonIdent( + "Bob Hacker", + "bob@example.com", + new Date(1500000600000L), + utc) + commitFile(git, dir, "a.txt", "one\n", alice, "first") + commitFile(git, dir, "b.txt", "two\n", bob, "second") + + val spreadsheet = new File(dir, "persons.xls") + val database = new File(dir, "persons.db") + unifyPersons.main(Array(dir.getPath, spreadsheet.getPath, database.getPath)) + + assert(spreadsheet.isFile) + assert(spreadsheet.length() > 0) + assert(database.isFile) + + Class.forName("org.sqlite.JDBC") + val connection = DriverManager.getConnection("jdbc:sqlite:" + database.getPath) + try { + def intQuery(sql: String): Int = { + val statement = connection.createStatement() + try { + val rows = statement.executeQuery(sql) + try { rows.next(); rows.getInt(1) } finally rows.close() + } finally statement.close() + } + + val statement = connection.prepareStatement( + "select autcount, comcount from emails where emailaddr = ?") + try { + statement.setString(1, "alice@example.com") + val rows = statement.executeQuery() + try { + assert(rows.next()) + assert(rows.getInt(1) === 1) + assert(rows.getInt(2) === 1) + } finally rows.close() + } finally statement.close() + + assert(intQuery("select count(*) from emails") === 2) + assert(intQuery("select count(*) from persons") === 2) + assert(intQuery("select count(*) from persons where personid = 'alice coder'") === 1) + } finally connection.close() + } + } } diff --git a/remapCommits/src/test/scala/remapCommitsSpec.scala b/remapCommits/src/test/scala/remapCommitsSpec.scala index 0c0e3f30..621a149e 100644 --- a/remapCommits/src/test/scala/remapCommitsSpec.scala +++ b/remapCommits/src/test/scala/remapCommitsSpec.scala @@ -17,6 +17,14 @@ along with this program. If not, see . import org.scalatest.FunSuite +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.lib.PersonIdent + +import java.io.{File, PrintWriter} +import java.nio.file.Files +import java.sql.DriverManager +import java.util.{Date, TimeZone} + class remapCommitsSpec extends FunSuite { val cid = "a" * 40 @@ -61,4 +69,77 @@ class remapCommitsSpec extends FunSuite { val message = "Former-commit-id: 0123456789abcdef" assert(remapCommits.extractOriginalCid(cid, message) === cid) } + + def withTempRepo(testCode: (Git, File) => Unit): Unit = { + val dir = Files.createTempDirectory("remapCommitsSpec").toFile + val git = Git.init.setDirectory(dir).call() + try { + testCode(git, dir) + } finally { + git.close() + def rm(file: File): Unit = { + if (file.isDirectory) file.listFiles.foreach(rm) + file.delete() + } + rm(dir) + } + } + + def commitFile( + git: Git, + dir: File, + name: String, + content: String, + who: PersonIdent, + message: String) = { + val out = new PrintWriter(new File(dir, name)) + out.print(content) + out.close() + git.add.addFilepattern(name).call() + git.commit.setAuthor(who).setCommitter(who).setMessage(message).call() + } + + test("main persists rewritten-to-original mappings in SQLite") { + withTempRepo { (git, dir) => + val who = new PersonIdent( + "Alice Coder", + "alice@example.com", + new Date(1500000000000L), + TimeZone.getTimeZone("UTC")) + val unchanged = commitFile(git, dir, "a.txt", "one\n", who, "first") + val rewritten = commitFile( + git, + dir, + "b.txt", + "two\n", + who, + s"second\n\nFormer-commit-id: $originalCid") + val database = new File(dir, "commit-map.db") + + remapCommits.main(Array(database.getPath, dir.getPath)) + + Class.forName("org.sqlite.JDBC") + val connection = DriverManager.getConnection("jdbc:sqlite:" + database.getPath) + try { + def mappedCid(cid: String): String = { + val statement = connection.prepareStatement( + "select originalcid from commitmap where cid = ?") + try { + statement.setString(1, cid) + val rows = statement.executeQuery() + try { assert(rows.next()); rows.getString(1) } finally rows.close() + } finally statement.close() + } + + val statement = connection.createStatement() + try { + val rows = statement.executeQuery("select count(*) from commitmap") + try { assert(rows.next()); assert(rows.getInt(1) === 2) } finally rows.close() + } finally statement.close() + + assert(mappedCid(unchanged.getName) === unchanged.getName) + assert(mappedCid(rewritten.getName) === originalCid) + } finally connection.close() + } + } } diff --git a/slickGitLog/src/test/scala/gitLogToDbSpec.scala b/slickGitLog/src/test/scala/gitLogToDbSpec.scala index a7a6849e..e236a753 100644 --- a/slickGitLog/src/test/scala/gitLogToDbSpec.scala +++ b/slickGitLog/src/test/scala/gitLogToDbSpec.scala @@ -23,6 +23,7 @@ import org.eclipse.jgit.lib.PersonIdent import java.io.File import java.io.PrintWriter import java.nio.file.Files +import java.sql.DriverManager import java.text.SimpleDateFormat import java.util.{Date, TimeZone} @@ -157,4 +158,49 @@ class gitLogToDbSpec extends FunSuite { assert(gitLogToDB.isBare(git) === false) } } + + test("main writes the complete commit metadata schema to SQLite") { + withTempRepo { (git, dir) => + val c1 = commitFile(git, dir, "a.txt", "one\n", alice, "first commit") + val c2 = commitFile( + git, + dir, + "b.txt", + "two\n", + bob, + "second commit\n\nSigned-off-by: Bob Hacker \n") + val dbFile = new File(dir, "history.db") + + gitLogToDB.main(Array(dbFile.getPath, dir.getPath)) + + Class.forName("org.sqlite.JDBC") + val connection = DriverManager.getConnection("jdbc:sqlite:" + dbFile.getPath) + try { + def intQuery(sql: String): Int = { + val statement = connection.createStatement() + try { + val rows = statement.executeQuery(sql) + try { rows.next(); rows.getInt(1) } finally rows.close() + } finally statement.close() + } + + def stringQuery(sql: String, parameter: String): String = { + val statement = connection.prepareStatement(sql) + try { + statement.setString(1, parameter) + val rows = statement.executeQuery() + try { assert(rows.next()); rows.getString(1) } finally rows.close() + } finally statement.close() + } + + assert(intQuery("select count(*) from commits") === 2) + assert(intQuery("select count(*) from parents") === 1) + assert(intQuery("select count(*) from logs") === 2) + assert(intQuery("select count(*) from footers") === 1) + assert(stringQuery("select summary from commits where cid = ?", c2.getName) === "second commit") + assert(stringQuery("select parent from parents where cid = ?", c2.getName) === c1.getName) + assert(stringQuery("select log from logs where cid = ?", c2.getName).contains("Signed-off-by")) + } finally connection.close() + } + } } From 4138d101e55cc4f280525732fc6c8324bf82dea7 Mon Sep 17 00:00:00 2001 From: Lucca Oliveira Date: Tue, 4 Aug 2026 11:32:57 -0300 Subject: [PATCH 08/12] test(prettyPrint): cover rendering and repository driver --- prettyPrint/t/prettyPrint-author.t | 134 ++++++++++++++++++++++++++ prettyPrint/t/prettyPrintFiles.t | 146 +++++++++++++++++++++++++++++ 2 files changed, 280 insertions(+) create mode 100644 prettyPrint/t/prettyPrint-author.t create mode 100644 prettyPrint/t/prettyPrintFiles.t diff --git a/prettyPrint/t/prettyPrint-author.t b/prettyPrint/t/prettyPrint-author.t new file mode 100644 index 00000000..a17e0ce0 --- /dev/null +++ b/prettyPrint/t/prettyPrint-author.t @@ -0,0 +1,134 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use Test::More; +use FindBin; +use File::Basename qw(dirname); +use File::Path qw(make_path); +use File::Temp qw(tempdir); +use DBI; + +my $script = "$FindBin::Bin/../prettyPrint-author.pl"; +my $workdir = tempdir(CLEANUP => 1); +my $cid = 'a' x 40; +my $original_cid = 'b' x 40; + +sub write_file { + my ($path, $content) = @_; + my $dir = dirname($path); + make_path($dir) unless -d $dir; + open(my $fh, '>', $path) or die "unable to write [$path]: $!"; + print $fh $content; + close $fh; +} + +sub slurp { + my ($path) = @_; + open(my $fh, '<', $path) or die "unable to read [$path]: $!"; + local $/; + my $content = <$fh>; + return defined $content ? $content : ''; +} + +my $cregit_db = "$workdir/cregit.db"; +my $authors_db = "$workdir/authors.db"; + +{ + my $dbh = DBI->connect("dbi:SQLite:dbname=$cregit_db", '', '', { RaiseError => 1 }); + $dbh->do('CREATE TABLE commits (cid TEXT PRIMARY KEY, autname TEXT, autemail TEXT, autdate TEXT, summary TEXT)'); + $dbh->do('CREATE TABLE commitmap (cid TEXT PRIMARY KEY, originalcid TEXT, repo TEXT)'); + $dbh->do( + 'INSERT INTO commits VALUES (?, ?, ?, ?, ?)', + undef, + $cid, + 'Alice Example', + 'alice@example.com', + '2020-01-01 00:00:00', + 'add "x"' + ); + $dbh->do('INSERT INTO commitmap VALUES (?, ?, ?)', undef, $cid, $original_cid, 'g'); + $dbh->disconnect(); +} + +{ + my $dbh = DBI->connect("dbi:SQLite:dbname=$authors_db", '', '', { RaiseError => 1 }); + $dbh->do('CREATE TABLE emails (emailname TEXT, emailaddr TEXT, personid TEXT)'); + $dbh->do('CREATE TABLE persons (personid TEXT PRIMARY KEY, personname TEXT)'); + $dbh->do( + 'INSERT INTO emails VALUES (?, ?, ?)', + undef, + 'Alice Example', + 'alice@example.com', + 'alice' + ); + $dbh->do('INSERT INTO persons VALUES (?, ?)', undef, 'alice', 'Alice Example'); + $dbh->disconnect(); +} + +my $source = "$workdir/example.c"; +my $blame = "$workdir/example.c.blame"; +my $bad_blame = "$workdir/bad.blame"; +my $header = "$workdir/header.html"; +my $footer = "$workdir/footer.html"; + +write_file($source, "int x;\n"); +write_file( + $blame, + "$cid;;\tname|int\n" . + "$cid;;\tname|x\n" . + "$cid;;\toperator|;\n" +); +write_file($bad_blame, "$cid;;\tname|float\n"); +write_file( + $header, + "HEADER _CREGIT_FILENAME_ _CREGIT_DIRNAME_ _CREGIT_VERSION_ _CREGIT_REPO_URL_\n" +); +write_file($footer, "FOOTER\n"); + +sub run_renderer { + my ($blame_file, $output) = @_; + my $stdout = "$workdir/stdout"; + my $stderr = "$workdir/stderr"; + my $status = system( + "'$^X' '$script' --header='$header' --footer='$footer' " . + "'$cregit_db' '$authors_db' '$source' '$blame_file' '$output' " . + "'src/example.c' 'https://example.test/commit/' > '$stdout' 2> '$stderr'" + ); + return ($status, slurp($stdout), slurp($stderr)); +} + +{ + my $output = "$workdir/out/deep/example.html"; + my ($status, $stdout, $stderr) = run_renderer($blame, $output); + is($status, 0, 'prettyPrint-author.pl renders a valid fixture'); + ok(-f $output, 'the HTML output is created, including parent directories'); + + my $html = slurp($output); + like( + $html, + qr/HEADER src\/example\.c src 1\.0-RC2 https:\/\/example\.test\/commit\//, + 'header placeholders are expanded' + ); + like($html, qr/FOOTER/, 'the configured footer is included'); + like($html, qr/Alice Example/, 'the unified author name comes from the persons database'); + like($html, qr/add "x"/, 'commit summaries are escaped in span metadata'); + like( + $html, + qr/windowpop\('$original_cid'\)/, + 'token links use the remapped original commit id' + ); + like($html, qr/Overall Contributors/, 'file-level contribution statistics are emitted'); + like($html, qr/3<\/td>/, 'all three source tokens are counted'); + is($stderr, '', 'a valid rendering has no stderr'); +} + +{ + my $output = "$workdir/out/bad.html"; + my ($status, $stdout, $stderr) = run_renderer($bad_blame, $output); + isnt($status, 0, 'a token/source mismatch fails the render'); + like($stderr, qr/Difference/, 'the mismatch explains the violated invariant'); + ok(!-e $output, 'a failed render does not publish a partial output file'); +} + +done_testing(); diff --git a/prettyPrint/t/prettyPrintFiles.t b/prettyPrint/t/prettyPrintFiles.t new file mode 100644 index 00000000..1c10205c --- /dev/null +++ b/prettyPrint/t/prettyPrintFiles.t @@ -0,0 +1,146 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use Test::More; +use FindBin; +use File::Basename qw(dirname); +use File::Path qw(make_path); +use File::Temp qw(tempdir); + +my $script = "$FindBin::Bin/../prettyPrintFiles.pl"; +my $workdir = tempdir(CLEANUP => 1); +my $repo = "$workdir/repo"; +my $blame_dir = "$workdir/blame"; +my $output_dir = "$workdir/html"; +my $call_log = "$workdir/calls.log"; + +sub write_file { + my ($path, $content) = @_; + my $dir = dirname($path); + make_path($dir) unless -d $dir; + open(my $fh, '>', $path) or die "unable to write [$path]: $!"; + print $fh $content; + close $fh; +} + +sub slurp { + my ($path) = @_; + return '' unless -f $path; + open(my $fh, '<', $path) or die "unable to read [$path]: $!"; + local $/; + my $content = <$fh>; + return defined $content ? $content : ''; +} + +sub git { + my (@args) = @_; + system('git', '-C', $repo, @args) == 0 or die "git failed: @args"; +} + +make_path($repo); +git('init', '-q', '-b', 'main'); +write_file("$repo/src/a.c", "int a;\n"); +write_file("$repo/src/b.c", "int b;\n"); +write_file("$repo/src/empty.c", ''); +write_file("$repo/notes.txt", "notes\n"); + +local $ENV{GIT_CONFIG_NOSYSTEM} = 1; +local $ENV{GIT_CONFIG_GLOBAL} = '/dev/null'; +local $ENV{GIT_AUTHOR_NAME} = 'Alice'; +local $ENV{GIT_AUTHOR_EMAIL} = 'alice@example.com'; +local $ENV{GIT_COMMITTER_NAME} = 'Alice'; +local $ENV{GIT_COMMITTER_EMAIL} = 'alice@example.com'; +git('add', '.'); +git('commit', '-q', '-m', 'fixture'); + +write_file("$blame_dir/src/a.c.blame", "fixture\n"); +write_file("$blame_dir/src/empty.c.blame", "fixture\n"); + +my $stub = "$workdir/pretty-stub.pl"; +write_file( + $stub, + <<'STUB' +#!/usr/bin/env perl +use strict; +use warnings; +use File::Basename qw(dirname); +use File::Path qw(make_path); + +open(my $log, '>>', $ENV{CALL_LOG}) or die $!; +print $log join("\t", @ARGV), "\n"; +close $log; + +my @positional = grep { $_ !~ /^--(?:header|footer)=/ } @ARGV; +my $output = $positional[4]; +my $dir = dirname($output); +make_path($dir) unless -d $dir; +open(my $out, '>', $output) or die $!; +print $out "rendered $positional[5]\n"; +close $out; +STUB +); +chmod 0755, $stub or die $!; + +my $header = "$workdir/header.html"; +my $footer = "$workdir/footer.html"; +write_file($header, "header\n"); +write_file($footer, "footer\n"); + +sub run_driver { + my (@extra) = @_; + my $stdout = "$workdir/stdout"; + my $stderr = "$workdir/stderr"; + local $ENV{CALL_LOG} = $call_log; + my $status = system( + "'$^X' '$script' --prettyCommand='$stub' --header='$header' --footer='$footer' " . + join(' ', @extra) . " 'cregit.db' 'authors.db' '$repo' '$blame_dir' " . + "'$output_dir' 'https://example.test/commit/' '\\.c\$' > '$stdout' 2> '$stderr'" + ); + return ($status, slurp($stdout), slurp($stderr)); +} + +{ + my ($status, $stdout, $stderr) = run_driver(); + is($status, 0, 'prettyPrintFiles.pl processes a repository fixture'); + ok(-f "$output_dir/src/a.c.html", 'a matching source with blame is rendered'); + ok(!-e "$output_dir/src/b.c.html", 'a source without blame is skipped'); + ok(!-e "$output_dir/src/empty.c.html", 'an empty source is skipped'); + ok(!-e "$output_dir/notes.txt.html", 'a non-matching extension is filtered out'); + like( + $stdout, + qr/Newly processed \[1\] Already done \[0\] files Error \[0\]/, + 'the first-run summary reports one generated file' + ); + is($stderr, '', 'the normal driver run has no stderr'); + + my $log = slurp($call_log); + like($log, qr/--header=\Q$header\E/, 'the custom header is forwarded'); + like($log, qr/--footer=\Q$footer\E/, 'the custom footer is forwarded'); + like($log, qr/src\/a\.c/, 'the repository-relative title is forwarded'); +} + +{ + my ($status, $stdout) = run_driver(); + is($status, 0, 'a second driver run succeeds'); + like( + $stdout, + qr/Newly processed \[0\] Already done \[1\] files Error \[0\]/, + 'existing HTML is skipped without --overwrite' + ); + is(scalar(() = slurp($call_log) =~ /rendered|--header=/g), 1, 'the renderer was called only once'); +} + +{ + my ($status, $stdout) = run_driver('--overwrite'); + is($status, 0, '--overwrite succeeds'); + like( + $stdout, + qr/Newly processed \[1\] Already done \[0\] files Error \[0\]/, + '--overwrite regenerates existing output' + ); + my @calls = grep { length } split /\n/, slurp($call_log); + is(scalar(@calls), 2, 'the renderer is invoked again during overwrite'); +} + +done_testing(); From b2f72bf87132c81f7499a35b86277a5f5108e49b Mon Sep 17 00:00:00 2001 From: Lucca Oliveira Date: Tue, 4 Aug 2026 11:33:05 -0300 Subject: [PATCH 09/12] fix(tokenize): skip empty simple-tokenizer fields --- tokenize/t/simpleTokenizer.t | 58 ++++++++++++++++++++++++++++++++ tokenize/text/simpleTokenizer.pl | 2 +- 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 tokenize/t/simpleTokenizer.t diff --git a/tokenize/t/simpleTokenizer.t b/tokenize/t/simpleTokenizer.t new file mode 100644 index 00000000..4d439da5 --- /dev/null +++ b/tokenize/t/simpleTokenizer.t @@ -0,0 +1,58 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use Test::More; +use FindBin; +use File::Temp qw(tempdir); + +my $script = "$FindBin::Bin/../text/simpleTokenizer.pl"; +my $workdir = tempdir(CLEANUP => 1); + +sub slurp { + my ($file) = @_; + open(my $fh, '<', $file) or die "unable to read [$file]: $!"; + local $/; + my $content = <$fh>; + return defined $content ? $content : ''; +} + +sub run_tokenizer { + my ($content, $use_file_argument) = @_; + my $input = "$workdir/input.txt"; + my $out = "$workdir/stdout"; + my $err = "$workdir/stderr"; + + open(my $fh, '>', $input) or die $!; + print $fh $content; + close $fh; + + my $redirect = $use_file_argument ? "'$input'" : "< '$input'"; + my $status = system("'$^X' '$script' $redirect > '$out' 2> '$err'"); + return ($status, slurp($out), slurp($err)); +} + +{ + my ($status, $out, $err) = run_tokenizer("alpha beta\ngamma\n", 0); + is($status, 0, 'tokenizing stdin succeeds'); + is($out, "alpha\nbeta\ngamma\n", 'runs of whitespace separate tokens'); + is($err, '', 'normal stdin tokenization has no stderr'); +} + +{ + my ($status, $out) = run_tokenizer("alpha (beta[gamma]) delta\n", 1); + is($status, 0, 'tokenizing a file argument succeeds'); + is( + $out, + "alpha\n(\nbeta\n[\ngamma\n]\n)\ndelta\n", + 'parentheses and brackets become standalone tokens' + ); +} + +{ + my ($status, $out) = run_tokenizer(" lone-token \n", 0); + is($status, 0, 'leading and trailing whitespace are accepted'); + is($out, "lone-token\n", 'whitespace does not emit empty tokens'); +} + +done_testing(); diff --git a/tokenize/text/simpleTokenizer.pl b/tokenize/text/simpleTokenizer.pl index 709e6777..862b489b 100755 --- a/tokenize/text/simpleTokenizer.pl +++ b/tokenize/text/simpleTokenizer.pl @@ -12,6 +12,7 @@ my @tokens = split(/\s+/, $s); for my $t (@tokens) { + next if $t eq ""; if ($t =~ /[(\[\]\)]/) { # print "++++$t+++++\n"; my @subst = split(/([(\[\]\)])/, $t); @@ -24,4 +25,3 @@ } } - From 336f3741a260402f667ced9739327e3507aa0cb7 Mon Sep 17 00:00:00 2001 From: Lucca Oliveira Date: Tue, 4 Aug 2026 11:34:10 -0300 Subject: [PATCH 10/12] docs: document the complete test matrix --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 89e2f892..f2ed5f67 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,29 @@ Use `make` to compile [srcMLtoken](./tokenize/srcMLtoken). Perl scripts can be run without compilation. +## How to test + +Run the test suites inside the pinned development environment (`devenv shell`). +The commands below mirror the required checks in GitHub Actions: + +```sh +cd blobExec && sbt -batch test assembly + +cd ../tokenize/srcMLtoken && make && make test +cd ../rustTokenizer && make && make test + +cd ../.. +prove tokenize/t tokenizeByBlobId/t blameRepo/t prettyPrint/t + +for module in slickGitLog persons remapCommits; do + (cd "$module" && sbt --java-home "$LEGACY_JAVA_HOME" -batch test one-jar) +done +``` + +The Perl tests create temporary Git repositories and SQLite databases. The +`tokenizeSrcMl` tests require `srcml2token`, so build the C++ tokenizer before +running `prove`. + ## How to use This is the workflow to process a git repository with cregit, and to generate the HTML views of its contributions. From 78a192f51091d25c9808f3463c8503582fe8b6a2 Mon Sep 17 00:00:00 2001 From: Lucca Oliveira Date: Wed, 12 Aug 2026 10:27:25 -0300 Subject: [PATCH 11/12] style(tests): remove redundant comments --- blameRepo/t/blameRepoFiles.t | 8 ----- blameRepo/t/formatBlame.t | 9 ----- persons/src/main/scala/unifyPersons.scala | 7 ---- persons/src/test/scala/unifyPersonsSpec.scala | 34 ++----------------- .../src/main/scala/remapCommits.scala | 2 -- .../src/test/scala/remapCommitsSpec.scala | 19 ----------- .../src/test/scala/gitLogToDbSpec.scala | 32 ++--------------- tokenize/t/tokenize.t | 10 ------ tokenize/t/tokenizeSrcMl.t | 23 ------------- tokenizeByBlobId/t/tokenBySha.t | 19 ----------- 10 files changed, 5 insertions(+), 158 deletions(-) diff --git a/blameRepo/t/blameRepoFiles.t b/blameRepo/t/blameRepoFiles.t index 1d4097af..4264c0c8 100644 --- a/blameRepo/t/blameRepoFiles.t +++ b/blameRepo/t/blameRepoFiles.t @@ -1,8 +1,5 @@ #!/usr/bin/env perl -# Tests for blameRepoFiles.pl: walks `git ls-files`, filters by regexp and -# runs formatBlame.pl per file, skipping files whose .blame already exists. - use strict; use warnings; use Test::More tests => 9; @@ -12,7 +9,6 @@ use File::Temp qw(tempdir); my $script = "$FindBin::Bin/../blameRepoFiles.pl"; my $workdir = tempdir(CLEANUP => 1); -# deterministic, hermetic git $ENV{GIT_CONFIG_NOSYSTEM} = 1; $ENV{GIT_CONFIG_GLOBAL} = '/dev/null'; $ENV{GIT_AUTHOR_DATE} = '2020-01-01T00:00:00 +0000'; @@ -35,7 +31,6 @@ sub write_file { close $fh; } -# fixture: two .c files and one .txt file in a single commit my $repo = "$workdir/repo"; mkdir $repo or die $!; git($repo, "init -q -b main"); @@ -48,7 +43,6 @@ git($repo, "commit -q -m first"); my $out = "$workdir/blame-out"; mkdir $out or die $!; -# first pass: both .c files processed, .txt filtered out { my $stdout = `perl '$script' '$repo' '$out' '\\.c\$' 2>'$workdir/stderr'`; is($?, 0, "blameRepoFiles.pl succeeds"); @@ -59,7 +53,6 @@ mkdir $out or die $!; "summary reports two newly processed files"); } -# second pass without --overwrite: everything is already done { my $stdout = `perl '$script' '$repo' '$out' '\\.c\$' 2>/dev/null`; is($?, 0, "second run succeeds"); @@ -67,7 +60,6 @@ mkdir $out or die $!; "existing .blame files are skipped"); } -# --overwrite reprocesses everything { my $stdout = `perl '$script' --overwrite '$repo' '$out' '\\.c\$' 2>/dev/null`; is($?, 0, "overwrite run succeeds"); diff --git a/blameRepo/t/formatBlame.t b/blameRepo/t/formatBlame.t index 1a1c36e0..8eee4ab8 100644 --- a/blameRepo/t/formatBlame.t +++ b/blameRepo/t/formatBlame.t @@ -1,9 +1,5 @@ #!/usr/bin/env perl -# Tests for formatBlame.pl: builds a small git repo on the fly (deterministic -# via GIT_AUTHOR_*/GIT_COMMITTER_* env vars) and checks the .blame output -# format: one line per source line, ";;\t". - use strict; use warnings; use Test::More tests => 12; @@ -13,7 +9,6 @@ use File::Temp qw(tempdir); my $script = "$FindBin::Bin/../formatBlame.pl"; my $workdir = tempdir(CLEANUP => 1); -# deterministic, hermetic git $ENV{GIT_CONFIG_NOSYSTEM} = 1; $ENV{GIT_CONFIG_GLOBAL} = '/dev/null'; $ENV{GIT_AUTHOR_DATE} = '2020-01-01T00:00:00 +0000'; @@ -53,7 +48,6 @@ sub slurp_lines { return @lines; } -# fixture: commit 1 (Alice) writes two lines, commit 2 (Bob) appends a third my $repo = "$workdir/repo"; mkdir $repo or die $!; git($repo, "init -q -b main"); @@ -64,7 +58,6 @@ write_file("$repo/f.c", "int one;\nint two;\nint three;\n"); git($repo, "add f.c"); my $cid2 = commit_as($repo, "Bob", "second"); -# basic blame formatting { my $dest = tempdir(CLEANUP => 1); my $status = system("perl '$script' '$repo' f.c '$dest' 2>'$workdir/stderr'"); @@ -78,7 +71,6 @@ my $cid2 = commit_as($repo, "Bob", "second"); is($lines[2], "$cid2;;\tint three;", "line 3 blamed on the second commit"); } -# a custom --blameExtension is honored { my $dest = tempdir(CLEANUP => 1); my $status = system("perl '$script' --blameExtension=.tok '$repo' f.c '$dest' 2>/dev/null"); @@ -86,7 +78,6 @@ my $cid2 = commit_as($repo, "Bob", "second"); ok(-f "$dest/f.c.tok", "creates /f.c.tok"); } -# after a rename, lines blamed on pre-rename commits carry the old filename { git($repo, "mv f.c g.c"); my $cid3 = commit_as($repo, "Alice", "rename"); diff --git a/persons/src/main/scala/unifyPersons.scala b/persons/src/main/scala/unifyPersons.scala index 6c23a196..b70b8f46 100644 --- a/persons/src/main/scala/unifyPersons.scala +++ b/persons/src/main/scala/unifyPersons.scala @@ -223,17 +223,12 @@ object unifyPersons { } def dealWithSingleWords(key:String, addon: String)= { - // we don't like names that don't have spaces - // since they are usually reused (eg. Jim, root, etc) - // so instead, use the other field val noacc = strip_accents(key) if (noacc.contains(' ')) noacc.toLowerCase else (noacc+" at " +addon).toLowerCase } - // unify by common email: merge groups of persons that share at least one - // lowercased email, transitively def unifyByEmail(setsNames: Iterable[Iterable[Person]]): Set[Set[Person]] = { setsNames.foldLeft(Set.empty[Set[Person]])((cum, curi) => { val cur = curi.toSet @@ -243,8 +238,6 @@ object unifyPersons { }) } - // we prefer names that contain a space (single words are usually - // reused, e.g. Jim, root); otherwise fall back to the email def preferredName(v: List[Person]): String = { if (v(0).name.contains(" ")) v(0).name else v(0).email } diff --git a/persons/src/test/scala/unifyPersonsSpec.scala b/persons/src/test/scala/unifyPersonsSpec.scala index 0fb01edf..664c0532 100644 --- a/persons/src/test/scala/unifyPersonsSpec.scala +++ b/persons/src/test/scala/unifyPersonsSpec.scala @@ -1,20 +1,3 @@ -/* - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -*/ - import org.scalatest.FunSuite import unifyPersons.Person @@ -34,8 +17,6 @@ class unifyPersonsSpec extends FunSuite { new Person(name, key, email, email.toLowerCase, user, domain) } - // strip_accents - test("strip_accents removes combining diacritical marks") { assert(unifyPersons.strip_accents("José") === "Jose") assert(unifyPersons.strip_accents("Éléonore Müller") === "Eleonore Muller") @@ -52,8 +33,6 @@ class unifyPersonsSpec extends FunSuite { assert(unifyPersons.strip_accents("John Smith") === "John Smith") } - // Person equality - test("Persons with identical fields are equal and hash-equal") { val a = mkPerson("Jim Smith", "jim@example.com") val b = mkPerson("Jim Smith", "jim@example.com") @@ -74,8 +53,6 @@ class unifyPersonsSpec extends FunSuite { assert(Set(a, b, c).size === 2) } - // splitEmail - test("splitEmail splits user and domain") { assert(unifyPersons.splitEmail("a@b.com") === ("a", "b.com")) } @@ -84,8 +61,6 @@ class unifyPersonsSpec extends FunSuite { assert(unifyPersons.splitEmail("localonly") === ("localonly", "")) } - // dealWithSingleWords - test("a name with a space becomes the lowercased name") { assert(unifyPersons.dealWithSingleWords("Jim Smith", "jim@x.com") === "jim smith") } @@ -98,15 +73,12 @@ class unifyPersonsSpec extends FunSuite { assert(unifyPersons.dealWithSingleWords("José Núñez", "jose@x.com") === "jose nunez") } - // unifyByEmail - test("unifyByEmail merges groups sharing an email, transitively") { val jim1 = mkPerson("Jim Smith", "jim@example.com") - val jim2 = mkPerson("James Smith", "jim@example.com") // shares email with jim1 - val jim3 = mkPerson("James Smith", "jsmith@work.org") // shares name-group with jim2 + val jim2 = mkPerson("James Smith", "jim@example.com") + val jim3 = mkPerson("James Smith", "jsmith@work.org") val ann = mkPerson("Ann Lee", "ann@example.org") - // groups as produced by the group-by-name step val groups = List(List(jim1), List(jim2, jim3), List(ann)) val unified = unifyPersons.unifyByEmail(groups) @@ -127,8 +99,6 @@ class unifyPersonsSpec extends FunSuite { assert(unifyPersons.unifyByEmail(Nil) === Set.empty[Set[Person]]) } - // preferredName - test("preferredName prefers a name containing a space") { val p = mkPerson("Jim Smith", "jim@example.com") assert(unifyPersons.preferredName(List(p)) === "Jim Smith") diff --git a/remapCommits/src/main/scala/remapCommits.scala b/remapCommits/src/main/scala/remapCommits.scala index fa883c0f..99d419bd 100644 --- a/remapCommits/src/main/scala/remapCommits.scala +++ b/remapCommits/src/main/scala/remapCommits.scala @@ -72,8 +72,6 @@ object remapCommits extends ProgramInfo { def extractOriginalCid(cid: String, message: String): String = { val lastline = - // split returns empty list if the string contains - // only separators. Weird. try { message.split("\n").last } catch { diff --git a/remapCommits/src/test/scala/remapCommitsSpec.scala b/remapCommits/src/test/scala/remapCommitsSpec.scala index 621a149e..ec9702f4 100644 --- a/remapCommits/src/test/scala/remapCommitsSpec.scala +++ b/remapCommits/src/test/scala/remapCommitsSpec.scala @@ -1,20 +1,3 @@ -/* - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -*/ - import org.scalatest.FunSuite import org.eclipse.jgit.api.Git @@ -54,8 +37,6 @@ class remapCommitsSpec extends FunSuite { } test("message of only newlines falls back to cid") { - // String.split("\n") on "\n" returns an empty array; .last throws and - // the catch turns it into "" -> no match -> cid assert(remapCommits.extractOriginalCid(cid, "\n") === cid) assert(remapCommits.extractOriginalCid(cid, "\n\n\n") === cid) } diff --git a/slickGitLog/src/test/scala/gitLogToDbSpec.scala b/slickGitLog/src/test/scala/gitLogToDbSpec.scala index e236a753..69969621 100644 --- a/slickGitLog/src/test/scala/gitLogToDbSpec.scala +++ b/slickGitLog/src/test/scala/gitLogToDbSpec.scala @@ -1,20 +1,3 @@ -/* - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -*/ - import org.scalatest.FunSuite import org.eclipse.jgit.api.Git @@ -32,9 +15,7 @@ class gitLogToDbSpec extends FunSuite { test("remove_trailing_space strips exactly one trailing space") { assert(gitLogToDB.remove_trailing_space("Bob ") === "Bob") assert(gitLogToDB.remove_trailing_space("Bob") === "Bob") - // regex is " $": only the last space is removed assert(gitLogToDB.remove_trailing_space("Bob ") === "Bob ") - // inner spaces are kept assert(gitLogToDB.remove_trailing_space("Bob Smith") === "Bob Smith") } @@ -44,8 +25,6 @@ class gitLogToDbSpec extends FunSuite { assert(gitLogToDB.parseGraftLine(s"$child $parent") === (parent, 1, child)) } - // --- fixture helpers ----------------------------------------------------- - def withTempRepo(testCode: (Git, File) => Unit): Unit = { val dir = Files.createTempDirectory("gitLogToDbSpec").toFile val git = Git.init.setDirectory(dir).call() @@ -73,12 +52,9 @@ class gitLogToDbSpec extends FunSuite { val utc = TimeZone.getTimeZone("UTC") val aliceDate = new Date(1500000000000L) val bobDate = new Date(1500000600000L) - // author name with a trailing space, to exercise remove_trailing_space val alice = new PersonIdent("Alice Coder ", "alice@example.com", aliceDate, utc) val bob = new PersonIdent("Bob Hacker", "bob@example.com", bobDate, utc) - // ------------------------------------------------------------------------- - test("git_commits_iterator maps commits to the expected tuples") { withTempRepo { (git, dir) => val c1 = commitFile(git, dir, "a.txt", "one\n", alice, "first commit") @@ -86,16 +62,14 @@ class gitLogToDbSpec extends FunSuite { "second commit\n\nSigned-off-by: Bob Hacker \n") val windows = gitLogToDB.git_commits_iterator(git).toList - assert(windows.size === 1) // 2 commits fit in one window of commitsPerOp + assert(windows.size === 1) - val commits = windows(0).sortBy(_._1._4) // order by author date + val commits = windows(0).sortBy(_._1._4) assert(commits.size === 2) val (commit1, log1, parents1, footers1) = commits(0) val (commit2, log2, parents2, footers2) = commits(1) - // expected dates computed with the same formatter over the same Date, - // so the assertion is immune to the JVM default timezone val dt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") assert(commit1 === (c1.getName, @@ -127,7 +101,7 @@ class gitLogToDbSpec extends FunSuite { val all = gitLogToDB.git_commits_iterator(git).toList.flatten val mergeTuple = all.find(_._1._1 == merge.getName).get - assert(mergeTuple._1._9 === true) // ismerge + assert(mergeTuple._1._9 === true) assert(mergeTuple._3 === Seq( (merge.getName, 0, main.getName), (merge.getName, 1, side.getName))) diff --git a/tokenize/t/tokenize.t b/tokenize/t/tokenize.t index 6fe1371d..b4b0f8ae 100644 --- a/tokenize/t/tokenize.t +++ b/tokenize/t/tokenize.t @@ -1,10 +1,5 @@ #!/usr/bin/env perl -# Tests for the tokenize.pl language dispatcher. It forwards to -# tokenizeSrcMl.pl (run with its built-in defaults, which is also how the -# dispatcher runs in production), so its output must be byte-identical to -# calling tokenizeSrcMl.pl directly with the same flags. - use strict; use warnings; use Test::More; @@ -32,7 +27,6 @@ sub slurp { return defined $content ? $content : ''; } -# returns (exit status, stdout, stderr) sub run_script { my ($script, @args) = @_; my $out = "$workdir/stdout"; @@ -41,8 +35,6 @@ sub run_script { return ($status, slurp($out), slurp($err)); } -# dispatch by explicit --language forwards flags and produces the same -# output as running tokenizeSrcMl.pl directly { my ($dstatus, $dout) = run_script($dispatcher, "--language=C", "--position", "'$fixtures/main.c'"); my ($sstatus, $sout) = run_script($direct, "--language=C", "--position", "'$fixtures/main.c'"); @@ -51,7 +43,6 @@ sub run_script { is($dout, $sout, "dispatcher output is byte-identical to the direct run"); } -# language autodetection from the file extension picks the same parser { my ($astatus, $aout) = run_script($dispatcher, "--position", "'$fixtures/main.c'"); my ($estatus, $eout) = run_script($dispatcher, "--language=C", "--position", "'$fixtures/main.c'"); @@ -59,7 +50,6 @@ sub run_script { is($aout, $eout, "autodetected output matches the explicit --language=C run"); } -# unknown --language is rejected { my ($status, $out, $err) = run_script($dispatcher, "--language=Nope", "'$fixtures/main.c'"); isnt($status, 0, "unknown --language exits non-zero"); diff --git a/tokenize/t/tokenizeSrcMl.t b/tokenize/t/tokenizeSrcMl.t index 3ecf750f..699b2fe0 100644 --- a/tokenize/t/tokenizeSrcMl.t +++ b/tokenize/t/tokenizeSrcMl.t @@ -1,23 +1,5 @@ #!/usr/bin/env perl -# Golden-file tests for tokenizeSrcMl.pl, mirroring the srcMLtoken pattern: -# run the tokenizer over committed fixtures and compare against t/expected/. -# -# Requires srcml and ctags on PATH and tokenize/srcMLtoken/srcml2token built -# (cd tokenize/srcMLtoken && make); skipped otherwise. -# -# To regenerate the goldens after an intentional output change, run inside -# the devenv shell, from the repo root: -# perl tokenize/tokenizeSrcMl.pl --ctags=ctags \ -# --srcml2token=$PWD/tokenize/srcMLtoken/srcml2token \ -# --position tokenize/srcMLtoken/tests/main.c tokenize/t/expected/main.c.token -# perl tokenize/tokenizeSrcMl.pl --ctags=ctags \ -# --srcml2token=$PWD/tokenize/srcMLtoken/srcml2token \ -# tokenize/srcMLtoken/tests/main.c tokenize/t/expected/main.c.nopos.token -# perl tokenize/tokenizeSrcMl.pl --ctags=ctags \ -# --srcml2token=$PWD/tokenize/srcMLtoken/srcml2token \ -# --position tokenize/srcMLtoken/tests/StringUtil.java tokenize/t/expected/StringUtil.java.token - use strict; use warnings; use Test::More; @@ -48,7 +30,6 @@ sub slurp { return defined $content ? $content : ''; } -# returns (exit status, stdout, stderr) sub run_tokenizer { my (@args) = @_; my $out = "$workdir/stdout"; @@ -58,7 +39,6 @@ sub run_tokenizer { return ($status, slurp($out), slurp($err)); } -# C fixture, with token positions { my ($status, $out, $err) = run_tokenizer("--position", "'$fixtures/main.c'"); is($status, 0, "tokenizing main.c with --position succeeds"); @@ -66,7 +46,6 @@ sub run_tokenizer { "main.c --position output matches the golden file"); } -# C fixture, without positions { my ($status, $out, $err) = run_tokenizer("'$fixtures/main.c'"); is($status, 0, "tokenizing main.c without --position succeeds"); @@ -74,7 +53,6 @@ sub run_tokenizer { "main.c output matches the golden file"); } -# Java fixture (language autodetected from the .java extension) { my ($status, $out, $err) = run_tokenizer("--position", "'$fixtures/StringUtil.java'"); is($status, 0, "tokenizing StringUtil.java succeeds"); @@ -82,7 +60,6 @@ sub run_tokenizer { "StringUtil.java output matches the golden file"); } -# unknown extension without --language is an error { my $bogus = "$workdir/mystery.xyz"; open(my $fh, '>', $bogus) or die $!; diff --git a/tokenizeByBlobId/t/tokenBySha.t b/tokenizeByBlobId/t/tokenBySha.t index 769b29c7..c98b51a6 100644 --- a/tokenizeByBlobId/t/tokenBySha.t +++ b/tokenizeByBlobId/t/tokenBySha.t @@ -1,14 +1,5 @@ #!/usr/bin/env perl -# Tests for tokenBySha.pl, the per-blob driver invoked by blobExec. It is -# argv-less by design: the blob arrives on stdin and everything else comes -# from BFG_* environment variables. A stub tokenizer stands in for the real -# tokenize command, so neither srcml nor ctags is needed. -# -# Note: the script anchors its scratch files to tokenizeByBlobId/build/ (it -# cleans them up itself); leftovers there from a failed run are scratch, not -# fixtures. - use strict; use warnings; use Test::More tests => 13; @@ -19,8 +10,6 @@ use Digest::SHA qw(sha1_hex); my $script = "$FindBin::Bin/../tokenBySha.pl"; my $workdir = tempdir(CLEANUP => 1); -# stub tokenizer: prints a banner, its --language argument, then the file -# it was given (the temp copy of the blob). Deterministic output only. my $stub = "$workdir/stub-tokenizer.sh"; { open(my $fh, '>', $stub) or die $!; @@ -37,8 +26,6 @@ sub slurp { return defined $content ? $content : ''; } -# runs tokenBySha.pl with the given env overrides and stdin content; -# returns (exit status, stdout, stderr) sub run_tokenbysha { my ($content, %env) = @_; my $in = "$workdir/stdin"; @@ -62,7 +49,6 @@ my $content = "int a;\nint b;\n"; my $sha1 = sha1_hex($content); my $memoFile = "$memoDir/" . substr($sha1, 0, 2) . "/" . substr($sha1, 2, 2) . "/$sha1"; -# first run: tokenizes via the stub and memoizes { my ($status, $out, $err) = run_tokenbysha($content, BFG_MEMO_DIR => $memoDir, @@ -79,7 +65,6 @@ my $memoFile = "$memoDir/" . substr($sha1, 0, 2) . "/" . substr($sha1, 2, 2) . " is(slurp($memoFile), $out, "memoized file matches stdout"); } -# second run: served from the memo without invoking the tokenize command { my ($status, $out, $err) = run_tokenbysha($content, BFG_MEMO_DIR => $memoDir, @@ -91,7 +76,6 @@ my $memoFile = "$memoDir/" . substr($sha1, 0, 2) . "/" . substr($sha1, 2, 2) . " is($out, slurp($memoFile), "cache hit replays the memoized output"); } -# extension mapping: .cpp maps to C++ { my ($status, $out, $err) = run_tokenbysha("class X {};\n", BFG_MEMO_DIR => $memoDir, @@ -103,7 +87,6 @@ my $memoFile = "$memoDir/" . substr($sha1, 0, 2) . "/" . substr($sha1, 2, 2) . " like($out, qr/^STUB-TOKENIZER\n--language=C\+\+\n/, ".cpp maps to --language=C++"); } -# unknown extension dies { my ($status, $out, $err) = run_tokenbysha($content, BFG_MEMO_DIR => $memoDir, @@ -115,7 +98,6 @@ my $memoFile = "$memoDir/" . substr($sha1, 0, 2) . "/" . substr($sha1, 2, 2) . " like($err, qr/unknown file extension/, "unknown extension reported on stderr"); } -# missing memo dir dies before doing any work { my ($status, $out, $err) = run_tokenbysha($content, BFG_MEMO_DIR => undef, @@ -126,7 +108,6 @@ my $memoFile = "$memoDir/" . substr($sha1, 0, 2) . "/" . substr($sha1, 2, 2) . " isnt($status, 0, "missing BFG_MEMO_DIR exits non-zero"); } -# empty BFG_FILENAME dies { my ($status, $out, $err) = run_tokenbysha($content, BFG_MEMO_DIR => $memoDir, From fab3825a0ec68f2c1d059dcd9f3233cb0cdf7d27 Mon Sep 17 00:00:00 2001 From: Lucca Oliveira Date: Sun, 16 Aug 2026 15:30:32 -0300 Subject: [PATCH 12/12] fix(pipeline): handle failures and metadata edge cases --- blameRepo/blameRepoFiles.pl | 1 + blameRepo/t/blameRepoFiles.t | 9 ++- persons/src/main/scala/unifyPersons.scala | 21 ++++-- persons/src/test/scala/unifyPersonsSpec.scala | 53 ++++++++++++-- prettyPrint/prettyPrintFiles.pl | 2 +- prettyPrint/t/prettyPrintFiles.t | 18 ++++- slickGitLog/src/main/scala/gitLogToDb.scala | 25 +++---- .../src/test/scala/gitLogToDbSpec.scala | 72 ++++++++++++++++++- tokenizeByBlobId/t/tokenBySha.t | 29 +++++++- tokenizeByBlobId/tokenBySha.pl | 13 +++- 10 files changed, 210 insertions(+), 33 deletions(-) diff --git a/blameRepo/blameRepoFiles.pl b/blameRepo/blameRepoFiles.pl index 7a6b3fad..1cbc0c95 100755 --- a/blameRepo/blameRepoFiles.pl +++ b/blameRepo/blameRepoFiles.pl @@ -104,6 +104,7 @@ } print "Newly processed [$count] Already done [$alreadyDone] files Error [$errorCount]\n"; +exit($errorCount == 0 ? 0 : 1); sub Usage { my ($m) = @_; diff --git a/blameRepo/t/blameRepoFiles.t b/blameRepo/t/blameRepoFiles.t index 4264c0c8..55650747 100644 --- a/blameRepo/t/blameRepoFiles.t +++ b/blameRepo/t/blameRepoFiles.t @@ -2,7 +2,7 @@ use strict; use warnings; -use Test::More tests => 9; +use Test::More tests => 11; use FindBin; use File::Temp qw(tempdir); @@ -66,3 +66,10 @@ mkdir $out or die $!; like($stdout, qr/Newly processed \[2\] Already done \[0\] files Error \[0\]/, "--overwrite reprocesses the files"); } + +{ + my $stdout = `perl '$script' --blameCommand=/bin/false --overwrite '$repo' '$out' '\\.c\$' 2>/dev/null`; + isnt($?, 0, "a formatter failure makes the repository driver fail"); + like($stdout, qr/Newly processed \[2\] Already done \[0\] files Error \[2\]/, + "the failure summary reports every formatter error"); +} diff --git a/persons/src/main/scala/unifyPersons.scala b/persons/src/main/scala/unifyPersons.scala index b70b8f46..5ead38ea 100644 --- a/persons/src/main/scala/unifyPersons.scala +++ b/persons/src/main/scala/unifyPersons.scala @@ -39,7 +39,7 @@ import impure._ import slick.driver.SQLiteDriver.api._ import java.io.File -import java.util.Calendar +import java.util.{Calendar, Locale} object unifyPersons { @@ -196,7 +196,14 @@ object unifyPersons { override def equals(that: Any): Boolean = that match { - case that: Person => that.canEqual(this) && this.hashCode == that.hashCode + case that: Person => + that.canEqual(this) && + name == that.name && + key == that.key && + email == that.email && + lcEmail == that.lcEmail && + lcUserId == that.lcUserId && + lcDomain == that.lcDomain case _ => false } override def hashCode:Int = { @@ -213,11 +220,11 @@ object unifyPersons { } def splitEmail(st:String) = { - val fields = st.split('@') + val fields = st.split("@", 2) if (fields.size > 1) { - (fields(0), fields(1)) + (fields(0).toLowerCase(Locale.ROOT), fields(1).toLowerCase(Locale.ROOT)) } else { - (fields(0), "") + (fields(0).toLowerCase(Locale.ROOT), "") } } @@ -289,8 +296,8 @@ object unifyPersons { Row(index) { Set( StringCell(0,key), - StringCell(1,p.name), - StringCell(2,p.key), + StringCell(1,p.key), + StringCell(2,p.name), StringCell(3,p.email), StringCell(4,p.lcUserId), StringCell(5,p.lcDomain), diff --git a/persons/src/test/scala/unifyPersonsSpec.scala b/persons/src/test/scala/unifyPersonsSpec.scala index 664c0532..ede7f3f8 100644 --- a/persons/src/test/scala/unifyPersonsSpec.scala +++ b/persons/src/test/scala/unifyPersonsSpec.scala @@ -3,8 +3,9 @@ import unifyPersons.Person import org.eclipse.jgit.api.Git import org.eclipse.jgit.lib.PersonIdent +import org.apache.poi.hssf.usermodel.HSSFWorkbook -import java.io.{File, PrintWriter} +import java.io.{File, FileInputStream, PrintWriter} import java.nio.file.Files import java.sql.DriverManager import java.util.{Date, TimeZone} @@ -46,6 +47,14 @@ class unifyPersonsSpec extends FunSuite { assert(a !== b) } + test("Persons with colliding hashes but different fields are not equal") { + val a = mkPerson("CCIB", "uhry@d49.test") + val b = mkPerson("XVHD", "u18rx@d23.test") + assert(a.hashCode === b.hashCode) + assert(a !== b) + assert(Set(a, b).size === 2) + } + test("equal Persons deduplicate in a Set") { val a = mkPerson("Jim Smith", "jim@example.com") val b = mkPerson("Jim Smith", "jim@example.com") @@ -55,6 +64,7 @@ class unifyPersonsSpec extends FunSuite { test("splitEmail splits user and domain") { assert(unifyPersons.splitEmail("a@b.com") === ("a", "b.com")) + assert(unifyPersons.splitEmail("Alice@Example.COM") === ("alice", "example.com")) } test("splitEmail without @ yields empty domain") { @@ -149,7 +159,7 @@ class unifyPersonsSpec extends FunSuite { val utc = TimeZone.getTimeZone("UTC") val alice = new PersonIdent( "Alice Coder", - "alice@example.com", + "Alice@Example.COM", new Date(1500000000000L), utc) val bob = new PersonIdent( @@ -165,9 +175,44 @@ class unifyPersonsSpec extends FunSuite { unifyPersons.main(Array(dir.getPath, spreadsheet.getPath, database.getPath)) assert(spreadsheet.isFile) - assert(spreadsheet.length() > 0) assert(database.isFile) + val spreadsheetInput = new FileInputStream(spreadsheet) + val workbook = new HSSFWorkbook(spreadsheetInput) + try { + assert(workbook.getNumberOfSheets === 2) + + val identities = workbook.getSheet("identities") + assert(identities != null) + assert(identities.getLastRowNum === 2) + assert((0 to 8).map(identities.getRow(0).getCell(_).getStringCellValue) === Seq( + "key", "lcname", "name", "email", "lcUserId", "lcDomain", + "countAll", "countAuthored", "countCommitted")) + assert((0 to 5).map(identities.getRow(1).getCell(_).getStringCellValue) === Seq( + "alice coder", "alice coder", "Alice Coder", "Alice@Example.COM", + "alice", "example.com")) + assert((6 to 8).map(identities.getRow(1).getCell(_).getNumericCellValue) === Seq(2.0, 1.0, 1.0)) + assert((0 to 5).map(identities.getRow(2).getCell(_).getStringCellValue) === Seq( + "bob hacker", "bob hacker", "Bob Hacker", "bob@example.com", + "bob", "example.com")) + assert((6 to 8).map(identities.getRow(2).getCell(_).getNumericCellValue) === Seq(2.0, 1.0, 1.0)) + + val stats = workbook.getSheet("stats") + assert(stats != null) + assert(stats.getLastRowNum === 2) + assert((0 to 5).map(stats.getRow(0).getCell(_).getStringCellValue) === Seq( + "key", "preferred", "identCount", "allCount", "authoredCount", "committedCount")) + assert((0 to 1).map(stats.getRow(1).getCell(_).getStringCellValue) === Seq( + "alice coder", "Alice Coder")) + assert((2 to 5).map(stats.getRow(1).getCell(_).getNumericCellValue) === Seq(1.0, 2.0, 1.0, 1.0)) + assert((0 to 1).map(stats.getRow(2).getCell(_).getStringCellValue) === Seq( + "bob hacker", "Bob Hacker")) + assert((2 to 5).map(stats.getRow(2).getCell(_).getNumericCellValue) === Seq(1.0, 2.0, 1.0, 1.0)) + } finally { + workbook.close() + spreadsheetInput.close() + } + Class.forName("org.sqlite.JDBC") val connection = DriverManager.getConnection("jdbc:sqlite:" + database.getPath) try { @@ -182,7 +227,7 @@ class unifyPersonsSpec extends FunSuite { val statement = connection.prepareStatement( "select autcount, comcount from emails where emailaddr = ?") try { - statement.setString(1, "alice@example.com") + statement.setString(1, "Alice@Example.COM") val rows = statement.executeQuery() try { assert(rows.next()) diff --git a/prettyPrint/prettyPrintFiles.pl b/prettyPrint/prettyPrintFiles.pl index 13612194..ef128ebd 100755 --- a/prettyPrint/prettyPrintFiles.pl +++ b/prettyPrint/prettyPrintFiles.pl @@ -141,7 +141,7 @@ } print "Newly processed [$count] Already done [$alreadyDone] files Error [$errorCount]\n"; -exit(0); +exit($errorCount == 0 ? 0 : 1); diff --git a/prettyPrint/t/prettyPrintFiles.t b/prettyPrint/t/prettyPrintFiles.t index 1c10205c..3c31c22c 100644 --- a/prettyPrint/t/prettyPrintFiles.t +++ b/prettyPrint/t/prettyPrintFiles.t @@ -38,6 +38,9 @@ sub git { system('git', '-C', $repo, @args) == 0 or die "git failed: @args"; } +local $ENV{GIT_CONFIG_NOSYSTEM} = 1; +local $ENV{GIT_CONFIG_GLOBAL} = '/dev/null'; + make_path($repo); git('init', '-q', '-b', 'main'); write_file("$repo/src/a.c", "int a;\n"); @@ -45,8 +48,6 @@ write_file("$repo/src/b.c", "int b;\n"); write_file("$repo/src/empty.c", ''); write_file("$repo/notes.txt", "notes\n"); -local $ENV{GIT_CONFIG_NOSYSTEM} = 1; -local $ENV{GIT_CONFIG_GLOBAL} = '/dev/null'; local $ENV{GIT_AUTHOR_NAME} = 'Alice'; local $ENV{GIT_AUTHOR_EMAIL} = 'alice@example.com'; local $ENV{GIT_COMMITTER_NAME} = 'Alice'; @@ -67,6 +68,8 @@ use warnings; use File::Basename qw(dirname); use File::Path qw(make_path); +exit 7 if $ENV{FAIL_RENDER}; + open(my $log, '>>', $ENV{CALL_LOG}) or die $!; print $log join("\t", @ARGV), "\n"; close $log; @@ -143,4 +146,15 @@ sub run_driver { is(scalar(@calls), 2, 'the renderer is invoked again during overwrite'); } +{ + local $ENV{FAIL_RENDER} = 1; + my ($status, $stdout) = run_driver('--overwrite'); + isnt($status, 0, 'a renderer failure makes the repository driver fail'); + like( + $stdout, + qr/Newly processed \[1\] Already done \[0\] files Error \[1\]/, + 'the failure summary reports the renderer error' + ); +} + done_testing(); diff --git a/slickGitLog/src/main/scala/gitLogToDb.scala b/slickGitLog/src/main/scala/gitLogToDb.scala index 29f808f9..409232b0 100644 --- a/slickGitLog/src/main/scala/gitLogToDb.scala +++ b/slickGitLog/src/main/scala/gitLogToDb.scala @@ -141,9 +141,12 @@ object gitLogToDB extends ProgramInfo { def remove_trailing_space(st:String) = st.replaceAll(" $", "") - def parseGraftLine(l: String): (String, Int, String) = { - val f = l.split(' ') - (f(1), 1, f(0)) + def parseGraftLine(l: String): Seq[(String, Int, String)] = { + val fields = l.trim.split("\\s+") + if (fields.length < 2) Seq.empty + else fields.tail.zipWithIndex.map { case (parent, idx) => + (fields.head, idx, parent) + } } def git_commits_iterator(git:Git) = { @@ -201,16 +204,12 @@ object gitLogToDB extends ProgramInfo { } def findGrafts(repo:String, git:Git) = { - val graftsFileName = repo + (if (isBare(git)) "" else "/.git/") + "info/grafts" - - // we assume that the heads of the grafts do not have any other parent... - // otherwise parent cannot be 1 - - // but why fix now? we might never run into that case - // we'll see + val graftsFile = new File(new File(git.getRepository.getDirectory, "info"), "grafts") - if ((new File(graftsFileName)).exists) { - Source.fromFile(graftsFileName).getLines.map(parseGraftLine).toList + if (graftsFile.exists) { + val source = Source.fromFile(graftsFile) + try source.getLines.flatMap(parseGraftLine).toList + finally source.close() } else { List() } @@ -304,7 +303,9 @@ object gitLogToDB extends ProgramInfo { println("Processing grafts...") + val graftedCommits = grafts.map(_._1).toSet val insertGr = DBIO.seq( + parents.filter(_.cid inSet graftedCommits).delete, parents ++= grafts) Await.result(db.run(insertGr), Duration.Inf) diff --git a/slickGitLog/src/test/scala/gitLogToDbSpec.scala b/slickGitLog/src/test/scala/gitLogToDbSpec.scala index 69969621..fa10f4e4 100644 --- a/slickGitLog/src/test/scala/gitLogToDbSpec.scala +++ b/slickGitLog/src/test/scala/gitLogToDbSpec.scala @@ -19,10 +19,20 @@ class gitLogToDbSpec extends FunSuite { assert(gitLogToDB.remove_trailing_space("Bob Smith") === "Bob Smith") } - test("parseGraftLine maps ' ' to (parent, 1, child)") { + test("parseGraftLine maps ' ' to a zero-indexed parent row") { val child = "c" * 40 val parent = "p" * 40 - assert(gitLogToDB.parseGraftLine(s"$child $parent") === (parent, 1, child)) + assert(gitLogToDB.parseGraftLine(s"$child $parent") === Seq((child, 0, parent))) + } + + test("parseGraftLine preserves every parent of a grafted merge") { + val child = "c" * 40 + val firstParent = "1" * 40 + val secondParent = "2" * 40 + assert(gitLogToDB.parseGraftLine( + s"$child $firstParent $secondParent") === Seq( + (child, 0, firstParent), + (child, 1, secondParent))) } def withTempRepo(testCode: (Git, File) => Unit): Unit = { @@ -117,7 +127,30 @@ class gitLogToDbSpec extends FunSuite { out.println(s"$child $parent") out.close() - assert(gitLogToDB.findGrafts(dir.getPath, git) === List((parent, 1, child))) + assert(gitLogToDB.findGrafts(dir.getPath, git) === List((child, 0, parent))) + } + } + + test("findGrafts reads info/grafts in a bare repo") { + val dir = Files.createTempDirectory("gitLogToDbBareSpec").toFile + val git = Git.init.setDirectory(dir).setBare(true).call() + try { + val child = "3" * 40 + val parent = "4" * 40 + new File(dir, "info").mkdirs() + val out = new PrintWriter(new File(dir, "info/grafts")) + out.println(s"$child $parent") + out.close() + + assert(gitLogToDB.isBare(git) === true) + assert(gitLogToDB.findGrafts(dir.getPath, git) === List((child, 0, parent))) + } finally { + git.close() + def rm(file: File): Unit = { + if (file.isDirectory) file.listFiles.foreach(rm) + file.delete() + } + rm(dir) } } @@ -133,6 +166,39 @@ class gitLogToDbSpec extends FunSuite { } } + test("main replaces recorded parents with graft parents") { + withTempRepo { (git, dir) => + val first = commitFile(git, dir, "a.txt", "one\n", alice, "first") + val second = commitFile(git, dir, "b.txt", "two\n", bob, "second") + val third = commitFile(git, dir, "c.txt", "three\n", alice, "third") + assert(third.getParent(0).getName === second.getName) + new File(dir, ".git/info").mkdirs() + val grafts = new PrintWriter(new File(dir, ".git/info/grafts")) + grafts.println(s"${third.getName} ${first.getName}") + grafts.close() + val dbFile = new File(dir, "grafted-history.db") + + gitLogToDB.main(Array(dbFile.getPath, dir.getPath)) + + Class.forName("org.sqlite.JDBC") + val connection = DriverManager.getConnection("jdbc:sqlite:" + dbFile.getPath) + try { + val statement = connection.prepareStatement( + "select idx, parent from parents where cid = ?") + try { + statement.setString(1, third.getName) + val rows = statement.executeQuery() + try { + assert(rows.next()) + assert(rows.getInt(1) === 0) + assert(rows.getString(2) === first.getName) + assert(!rows.next()) + } finally rows.close() + } finally statement.close() + } finally connection.close() + } + } + test("main writes the complete commit metadata schema to SQLite") { withTempRepo { (git, dir) => val c1 = commitFile(git, dir, "a.txt", "one\n", alice, "first commit") diff --git a/tokenizeByBlobId/t/tokenBySha.t b/tokenizeByBlobId/t/tokenBySha.t index c98b51a6..5dd7d30f 100644 --- a/tokenizeByBlobId/t/tokenBySha.t +++ b/tokenizeByBlobId/t/tokenBySha.t @@ -2,7 +2,7 @@ use strict; use warnings; -use Test::More tests => 13; +use Test::More tests => 18; use FindBin; use File::Temp qw(tempdir); use Digest::SHA qw(sha1_hex); @@ -87,6 +87,33 @@ my $memoFile = "$memoDir/" . substr($sha1, 0, 2) . "/" . substr($sha1, 2, 2) . " like($out, qr/^STUB-TOKENIZER\n--language=C\+\+\n/, ".cpp maps to --language=C++"); } +{ + my $failedContent = "int failure;\n"; + my $failedSha1 = sha1_hex($failedContent); + my $failedMemoFile = "$memoDir/" . substr($failedSha1, 0, 2) . "/" . + substr($failedSha1, 2, 2) . "/$failedSha1"; + + my ($status, $out, $err) = run_tokenbysha($failedContent, + BFG_MEMO_DIR => $memoDir, + BFG_TOKENIZE_CMD => "/bin/false", + BFG_BLOB => "5" x 40, + BFG_FILENAME => "failure.c", + ); + isnt($status, 0, "a tokenizer failure on a cache miss exits non-zero"); + like($err, qr/tokenize command failed/, "the tokenizer failure is reported"); + ok(!-e $failedMemoFile, "failed tokenizer output is not memoized"); + + ($status, $out, $err) = run_tokenbysha($failedContent, + BFG_MEMO_DIR => $memoDir, + BFG_TOKENIZE_CMD => $stub, + BFG_BLOB => "5" x 40, + BFG_FILENAME => "failure.c", + ); + is($status, 0, "the same blob can be tokenized after the command is fixed"); + is($out, "STUB-TOKENIZER\n--language=C\n$failedContent", + "the retry executes the fixed tokenizer instead of replaying a poisoned cache entry"); +} + { my ($status, $out, $err) = run_tokenbysha($content, BFG_MEMO_DIR => $memoDir, diff --git a/tokenizeByBlobId/tokenBySha.pl b/tokenizeByBlobId/tokenBySha.pl index 60464fea..f42967e1 100755 --- a/tokenizeByBlobId/tokenBySha.pl +++ b/tokenizeByBlobId/tokenBySha.pl @@ -137,8 +137,17 @@ print $_; print $fout $_; } - close PROC; - close $fout; + my $tokenizeSucceeded = close PROC; + my $tokenizeStatus = $?; + close $fout or die "unable to close temporary tokenizer output [$outfile]: $!"; + + if (not $tokenizeSucceeded) { + unlink($outfile); + unlink($file); + my $exitCode = $tokenizeStatus == -1 ? -1 : $tokenizeStatus >> 8; + die "tokenize command failed with exit code [$exitCode] [$tokenizeCmd]"; + } + if (not -d $dir) { make_path($dir); }