test(pipeline): add Scala and Perl coverage and fix discovered defects

.github/workflows/ci.yml

@@ -56,21 +56,24 @@ jobs:
           restore-keys: |
             cargo-${{ runner.os }}-
 
-      - name: Build blobExec (sbt assembly, Scala 2.13 / JDK 21)
-        run: devenv shell -- bash -c 'cd blobExec && sbt -batch assembly'
+      - name: Build & test blobExec (sbt test assembly, Scala 2.13 / JDK 21)
+        run: devenv shell -- bash -c 'cd blobExec && sbt -batch test assembly'
 
       - name: Build & test C++ transcoder (srcml2token)
         run: devenv shell -- bash -c 'cd tokenize/srcMLtoken && make && make test'
 
       - name: Build & test Rust tokenizer (rust_tokenizer)
         run: devenv shell -- bash -c 'cd tokenize/rustTokenizer && make && make test'
 
-      - name: Build legacy onejar modules (sbt one-jar, Scala 2.10 / JDK 8)
+      - name: Perl tests (prove)
+        run: devenv shell -- bash -c 'prove tokenize/t tokenizeByBlobId/t blameRepo/t prettyPrint/t'
+
+      - name: Build & test legacy onejar modules (sbt test one-jar, Scala 2.10 / JDK 8)
         run: |
           devenv shell -- bash -c '
             set -e
             for m in slickGitLog persons remapCommits; do
               echo "::group::$m"
-              ( cd "$m" && sbt --java-home "$LEGACY_JAVA_HOME" -batch one-jar )
+              ( cd "$m" && sbt --java-home "$LEGACY_JAVA_HOME" -batch test one-jar )
               echo "::endgroup::"
             done'

README.md

@@ -52,6 +52,29 @@ Use `make` to compile [srcMLtoken](./tokenize/srcMLtoken).
 
 Perl scripts can be run without compilation.
 
+## How to test
+
+Run the test suites inside the pinned development environment (`devenv shell`).
+The commands below mirror the required checks in GitHub Actions:
+
+```sh
+cd blobExec && sbt -batch test assembly
+
+cd ../tokenize/srcMLtoken && make && make test
+cd ../rustTokenizer && make && make test
+
+cd ../..
+prove tokenize/t tokenizeByBlobId/t blameRepo/t prettyPrint/t
+
+for module in slickGitLog persons remapCommits; do
+  (cd "$module" && sbt --java-home "$LEGACY_JAVA_HOME" -batch test one-jar)
+done
+```
+
+The Perl tests create temporary Git repositories and SQLite databases. The
+`tokenizeSrcMl` tests require `srcml2token`, so build the C++ tokenizer before
+running `prove`.
+
 ## How to use
 
 This is the workflow to process a git repository with cregit, and to generate the HTML views of its contributions.

blameRepo/blameRepoFiles.pl

@@ -104,6 +104,7 @@
 }
 
 print "Newly processed [$count] Already done [$alreadyDone] files Error [$errorCount]\n";
+exit($errorCount == 0 ? 0 : 1);
 
 sub Usage {
     my ($m) = @_;

blameRepo/t/blameRepoFiles.t

@@ -0,0 +1,75 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+use Test::More tests => 11;
+use FindBin;
+use File::Temp qw(tempdir);
+
+my $script  = "$FindBin::Bin/../blameRepoFiles.pl";
+my $workdir = tempdir(CLEANUP => 1);
+
+$ENV{GIT_CONFIG_NOSYSTEM} = 1;
+$ENV{GIT_CONFIG_GLOBAL}   = '/dev/null';
+$ENV{GIT_AUTHOR_DATE}     = '2020-01-01T00:00:00 +0000';
+$ENV{GIT_COMMITTER_DATE}  = '2020-01-01T00:00:00 +0000';
+$ENV{GIT_AUTHOR_NAME}     = 'Alice';
+$ENV{GIT_AUTHOR_EMAIL}    = 'alice@example.com';
+$ENV{GIT_COMMITTER_NAME}  = 'Alice';
+$ENV{GIT_COMMITTER_EMAIL} = 'alice@example.com';
+
+sub git {
+    my ($repo, @args) = @_;
+    my $cmd = "git -C '$repo' " . join(' ', @args);
+    system($cmd) == 0 or die "git failed: $cmd";
+}
+
+sub write_file {
+    my ($path, $content) = @_;
+    open(my $fh, '>', $path) or die $!;
+    print $fh $content;
+    close $fh;
+}
+
+my $repo = "$workdir/repo";
+mkdir $repo or die $!;
+git($repo, "init -q -b main");
+write_file("$repo/a.c", "int a;\n");
+write_file("$repo/b.c", "int b;\n");
+write_file("$repo/notes.txt", "hello\n");
+git($repo, "add .");
+git($repo, "commit -q -m first");
+
+my $out = "$workdir/blame-out";
+mkdir $out or die $!;
+
+{
+    my $stdout = `perl '$script' '$repo' '$out' '\\.c\$' 2>'$workdir/stderr'`;
+    is($?, 0, "blameRepoFiles.pl succeeds");
+    ok(-f "$out/a.c.blame", "a.c.blame created");
+    ok(-f "$out/b.c.blame", "b.c.blame created");
+    ok(!-e "$out/notes.txt.blame", "notes.txt is filtered out by the regexp");
+    like($stdout, qr/Newly processed \[2\] Already done \[0\] files Error \[0\]/,
+         "summary reports two newly processed files");
+}
+
+{
+    my $stdout = `perl '$script' '$repo' '$out' '\\.c\$' 2>/dev/null`;
+    is($?, 0, "second run succeeds");
+    like($stdout, qr/Newly processed \[0\] Already done \[2\] files Error \[0\]/,
+         "existing .blame files are skipped");
+}
+
+{
+    my $stdout = `perl '$script' --overwrite '$repo' '$out' '\\.c\$' 2>/dev/null`;
+    is($?, 0, "overwrite run succeeds");
+    like($stdout, qr/Newly processed \[2\] Already done \[0\] files Error \[0\]/,
+         "--overwrite reprocesses the files");
+}
+
+{
+    my $stdout = `perl '$script' --blameCommand=/bin/false --overwrite '$repo' '$out' '\\.c\$' 2>/dev/null`;
+    isnt($?, 0, "a formatter failure makes the repository driver fail");
+    like($stdout, qr/Newly processed \[2\] Already done \[0\] files Error \[2\]/,
+         "the failure summary reports every formatter error");
+}

blameRepo/t/formatBlame.t

@@ -0,0 +1,95 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+use Test::More tests => 12;
+use FindBin;
+use File::Temp qw(tempdir);
+
+my $script  = "$FindBin::Bin/../formatBlame.pl";
+my $workdir = tempdir(CLEANUP => 1);
+
+$ENV{GIT_CONFIG_NOSYSTEM} = 1;
+$ENV{GIT_CONFIG_GLOBAL}   = '/dev/null';
+$ENV{GIT_AUTHOR_DATE}     = '2020-01-01T00:00:00 +0000';
+$ENV{GIT_COMMITTER_DATE}  = '2020-01-01T00:00:00 +0000';
+$ENV{GIT_COMMITTER_NAME}  = 'Committer';
+$ENV{GIT_COMMITTER_EMAIL} = 'committer@example.com';
+
+sub git {
+    my ($repo, @args) = @_;
+    my $cmd = "git -C '$repo' " . join(' ', @args);
+    my $out = `$cmd`;
+    die "git failed: $cmd" if $? != 0;
+    chomp $out;
+    return $out;
+}
+
+sub commit_as {
+    my ($repo, $name, $message) = @_;
+    local $ENV{GIT_AUTHOR_NAME}  = $name;
+    local $ENV{GIT_AUTHOR_EMAIL} = lc($name) . '@example.com';
+    git($repo, "commit -q -m '$message'");
+    return git($repo, "rev-parse HEAD");
+}
+
+sub write_file {
+    my ($path, $content) = @_;
+    open(my $fh, '>', $path) or die $!;
+    print $fh $content;
+    close $fh;
+}
+
+sub slurp_lines {
+    my ($file) = @_;
+    open(my $fh, '<', $file) or die "unable to read [$file]: $!";
+    my @lines = <$fh>;
+    chomp @lines;
+    return @lines;
+}
+
+my $repo = "$workdir/repo";
+mkdir $repo or die $!;
+git($repo, "init -q -b main");
+write_file("$repo/f.c", "int one;\nint two;\n");
+git($repo, "add f.c");
+my $cid1 = commit_as($repo, "Alice", "first");
+write_file("$repo/f.c", "int one;\nint two;\nint three;\n");
+git($repo, "add f.c");
+my $cid2 = commit_as($repo, "Bob", "second");
+
+{
+    my $dest = tempdir(CLEANUP => 1);
+    my $status = system("perl '$script' '$repo' f.c '$dest' 2>'$workdir/stderr'");
+    is($status, 0, "formatBlame.pl succeeds");
+    ok(-f "$dest/f.c.blame", "creates <dest>/f.c.blame");
+
+    my @lines = slurp_lines("$dest/f.c.blame");
+    is(scalar(@lines), 3, "one blame line per source line");
+    is($lines[0], "$cid1;;\tint one;",   "line 1 blamed on the first commit");
+    is($lines[1], "$cid1;;\tint two;",   "line 2 blamed on the first commit");
+    is($lines[2], "$cid2;;\tint three;", "line 3 blamed on the second commit");
+}
+
+{
+    my $dest = tempdir(CLEANUP => 1);
+    my $status = system("perl '$script' --blameExtension=.tok '$repo' f.c '$dest' 2>/dev/null");
+    is($status, 0, "formatBlame.pl with --blameExtension succeeds");
+    ok(-f "$dest/f.c.tok", "creates <dest>/f.c.tok");
+}
+
+{
+    git($repo, "mv f.c g.c");
+    my $cid3 = commit_as($repo, "Alice", "rename");
+
+    my $dest = tempdir(CLEANUP => 1);
+    my $status = system("perl '$script' '$repo' g.c '$dest' 2>/dev/null");
+    is($status, 0, "formatBlame.pl on the renamed file succeeds");
+
+    my @lines = slurp_lines("$dest/g.c.blame");
+    is(scalar(@lines), 3, "renamed file still has three blame lines");
+    is($lines[0], "$cid1;f.c;\tint one;",
+       "pre-rename lines carry the original filename");
+    is($lines[2], "$cid2;f.c;\tint three;",
+       "all pre-rename commits report the old name");
+}

persons/build.sbt

@@ -7,7 +7,8 @@ libraryDependencies ++= Seq(
   "org.xerial" % "sqlite-jdbc" % "3.45.3.0",
   "com.zaxxer" % "HikariCP" % "2.4.1",
   "org.eclipse.jgit" % "org.eclipse.jgit" % "4.6.0.201612231935-r",
-  "info.folone" %% "poi-scala" % "0.18"
+  "info.folone" %% "poi-scala" % "0.18",
+  "org.scalatest" %% "scalatest" % "3.0.8" % "test"
 )
 
 resolvers ++= Seq(

persons/src/main/scala/unifyPersons.scala

@@ -39,7 +39,7 @@ import impure._
 
 import slick.driver.SQLiteDriver.api._
 import java.io.File
-import java.util.Calendar 
+import java.util.{Calendar, Locale}
 
 
 object  unifyPersons {
@@ -196,7 +196,14 @@ object  unifyPersons {
 
     override def equals(that: Any): Boolean =
       that match {
-        case that: Person => that.canEqual(this) && this.hashCode == that.hashCode
+        case that: Person =>
+          that.canEqual(this) &&
+          name == that.name &&
+          key == that.key &&
+          email == that.email &&
+          lcEmail == that.lcEmail &&
+          lcUserId == that.lcUserId &&
+          lcDomain == that.lcDomain
         case _ => false
       }
     override def hashCode:Int = {
@@ -212,6 +219,36 @@ object  unifyPersons {
     }
   }
 
+  def splitEmail(st:String) = {
+    val fields = st.split("@", 2)
+    if (fields.size > 1) {
+      (fields(0).toLowerCase(Locale.ROOT), fields(1).toLowerCase(Locale.ROOT))
+    } else {
+      (fields(0).toLowerCase(Locale.ROOT), "")
+    }
+
+  }
+
+  def dealWithSingleWords(key:String, addon: String)= {
+    val noacc = strip_accents(key)
+    if (noacc.contains(' '))
+      noacc.toLowerCase
+    else (noacc+" at " +addon).toLowerCase
+  }
+
+  def unifyByEmail(setsNames: Iterable[Iterable[Person]]): Set[Set[Person]] = {
+    setsNames.foldLeft(Set.empty[Set[Person]])((cum, curi) => {
+      val cur = curi.toSet
+      val curEmails = cur.map{_.lcEmail}
+      val (hasCommon, rest) = cum.partition(_.map{_.lcEmail} & curEmails nonEmpty)
+      rest + (cur ++ hasCommon.flatten)
+    })
+  }
+
+  def preferredName(v: List[Person]): String = {
+    if (v(0).name.contains(" ")) v(0).name else v(0).email
+  }
+
   // return an iterator that returns, for each commit
   // a tuple of the author  and the committer info
   def git_commits_iterator(git:Git) = {
@@ -220,16 +257,6 @@ object  unifyPersons {
 
     val logsIt = logs.asScala.toIterator
 
-    def splitEmail(st:String) = {
-      val fields = st.split('@')
-      if (fields.size > 1) {
-        (fields(0), fields(1))
-      } else {
-        (fields(0), "")
-      }
-
-    }
-
     logsIt.map { l =>
       val author = l.getAuthorIdent().getEmailAddress
       val committer = l.getCommitterIdent().getEmailAddress
@@ -239,16 +266,6 @@ object  unifyPersons {
       val authorName = l.getAuthorIdent().getName
       val committerName = l.getCommitterIdent().getName
 
-      def dealWithSingleWords(key:String, addon: String)= {
-        // we don't like names that don't have spaces
-        // since they are usually reused (eg. Jim, root, etc)
-        // so instead, use the other field
-        val noacc = strip_accents(key)
-        if (noacc.contains(' '))
-          noacc.toLowerCase
-        else (noacc+" at " +addon).toLowerCase
-      }
-
       val authorKey = dealWithSingleWords(authorName, author)
       val commKey = dealWithSingleWords(committerName, committer)
 
@@ -279,8 +296,8 @@ object  unifyPersons {
       Row(index) {
         Set(
           StringCell(0,key),
-          StringCell(1,p.name),
-          StringCell(2,p.key),
+          StringCell(1,p.key),
+          StringCell(2,p.name),
           StringCell(3,p.email),
           StringCell(4,p.lcUserId),
           StringCell(5,p.lcDomain),
@@ -426,12 +443,7 @@ object  unifyPersons {
     println("Unifying by email...")
 
     // unify by common email
-    val unifiedByEmail = setsNames.foldLeft(Set.empty[Set[Person]])((cum, curi) => {
-      val cur = curi.toSet
-      val curEmails = cur.map{_.lcEmail}
-      val (hasCommon, rest) = cum.partition(_.map{_.lcEmail} & curEmails nonEmpty)
-      rest + (cur ++ hasCommon.flatten)
-    })
+    val unifiedByEmail = unifyByEmail(setsNames)
 
     println(s"    ... reduced to ${unifiedByEmail.size} emails")
 
@@ -455,7 +467,7 @@ object  unifyPersons {
     // attach the count of all, authored, committed
 
     val keys = mapByKey.map{ case (k,v) =>
-      val nameToUse = if (v(0).name.contains(" ")) v(0).name else v(0).email
+      val nameToUse = preferredName(v)
       val identCount = v.size
       val countAll = v.map{ e =>
         (everybody(e),