process-transcripts.js

#!/usr/bin/env node

const fs = require('fs');
const path = require('path');
const { execFileSync } = require('child_process');
const { ROCrate } = require('ro-crate');
const Workbook = require('ro-crate-excel/lib/workbook.js');
const { unicodeName } = require('unicode-name');

function printUsage() {
  console.log(`Usage: node process-transcripts.js --input <dir> --output <dir> [options]

Options:
  --input, -i        Directory containing .docx transcript files (default: ./input)
  --output, -o       Directory for generated CSV outputs (default: ./output)
  --header-rows      Number of leading rows to remove from each transcript (default: 0)
  --footer-rows      Number of trailing rows to remove from each transcript (default: 0)
  --pre-start        Start index for PRE rows (optional)
  --pre-end          End index for PRE rows (optional)
  --post-start       Start index for POST rows (optional)
  --post-end         End index for POST rows (optional)
  --help, -h         Show this help message
`);
}

function parseArgs(argv) {
  const config = {
    input: './input',
    output: './output',
    headerRows: 0,
    footerRows: 0,
    preStart: null,
    preEnd: null,
    postStart: null,
    postEnd: null,
  };

  for (let i = 0; i < argv.length; i += 1) {
    const arg = argv[i];

    switch (arg) {
      case '--help':
      case '-h':
        printUsage();
        process.exit(0);
      case '--input':
      case '-i':
        config.input = argv[++i];
        break;
      case '--output':
      case '-o':
        config.output = argv[++i];
        break;
      case '--header-rows':
        config.headerRows = Number(argv[++i] || 0);
        break;
      case '--footer-rows':
        config.footerRows = Number(argv[++i] || 0);
        break;
      case '--pre-start':
        config.preStart = Number(argv[++i] || 0);
        break;
      case '--pre-end':
        config.preEnd = Number(argv[++i] || 0);
        break;
      case '--post-start':
        config.postStart = Number(argv[++i] || 0);
        break;
      case '--post-end':
        config.postEnd = Number(argv[++i] || 0);
        break;
      default:
        throw new Error(`Unknown argument: ${arg}`);
    }
  }

  return config;
}

function ensurePythonDocxInstalled() {
  try {
    execFileSync('python3', ['-c', 'import docx'], { stdio: 'ignore' });
    return true;
  } catch (error) {
    return false;
  }
}

function extractParagraphs(docxPath) {
  const result = execFileSync(
    'python3',
    [
      '-c',
      `
import json, sys
from docx import Document

doc = Document(sys.argv[1])
print(json.dumps([p.text for p in doc.paragraphs]))
`,
      docxPath,
    ],
    { encoding: 'utf8' }
  );

  return JSON.parse(result);
}

function normalizeText(text) {
  let normalized = String(text || '');

  normalized = normalized.replace(/\r\n/g, '\n');
  normalized = normalized.replace(/\r/g, '\n');
  normalized = normalized.replace(/\u00A0/g, ' ');

  normalized = normalized.replace(/^[\t ]+/gm, '\t');
  normalized = normalized.replace(/^([A-Z]):[\t ]+/gm, '$1:\t');
  normalized = normalized.replace(/^([A-Z])[\t ]+/gm, '$1:\t');
  normalized = normalized.replace(/(\t.*) \t/gm, '$1 ');
  normalized = normalized.replace(/^.*END OF TRANSCRIPT.*$/gm, '');

  return normalized;
}

function mergeContinuationLines(text) {
  let merged = text;
  const protectedLines = new Set([
    'Speakers:',
    'PRELIMINARIES',
    'MAIN',
    'POSTLIMINARIES',
    'Transcript:',
    'Recording date:',
    'Length of audio recording:',
    'Length of video recording:',
    'Transcriber:',
  ]);

  for (let iteration = 0; iteration < 100; iteration += 1) {
    const lines = merged.split('\n');
    const repaired = [];
    let changed = false;

    for (let i = 0; i < lines.length; i += 1) {
      const line = lines[i];
      const trimmed = line.trim();
      const isSpeakerLine = /^([A-Z][A-Z0-9]?\s*:|[A-Z][A-Z0-9]?:)\s*(\t|.*)$/.test(line);

      if (protectedLines.has(trimmed)) {
        repaired.push(line);
        continue;
      }

      if (!isSpeakerLine && repaired.length > 0) {
        const previous = repaired[repaired.length - 1];
        const nextValue = previous.trimEnd() + ' ' + line.trim();
        repaired[repaired.length - 1] = nextValue;
        changed = true;
      } else {
        repaired.push(line);
      }
    }

    const candidate = repaired.join('\n');
    if (!changed || candidate === merged) {
      merged = candidate;
      break;
    }
    merged = candidate;
  }

  return merged;
}

function parseSpeakerBlock(lines, warnings) {
  const speakers = new Map();
  let inSpeakerSection = false;

  for (const line of lines) {
    const trimmed = line.trim();
    if (!trimmed) {
      continue;
    }

    if (trimmed === 'Speakers:') {
      inSpeakerSection = true;
      continue;
    }

    if (inSpeakerSection && trimmed === 'PRELIMINARIES') {
      inSpeakerSection = false;
      break;
    }

    if (!inSpeakerSection) {
      continue;
    }

    const speakerMatch = trimmed.match(/^([A-Z][A-Z0-9]?)\s*:\s*(.*)$/);
    if (!speakerMatch) {
      continue;
    }

    const speakerID = speakerMatch[1];
    const speakerText = speakerMatch[2].trim();
    const optionalCode = speakerText.match(/(#\S+)/)?.[1] ?? null;
    const resolvedSpeakerID = optionalCode || speakerID;
    speakers.set(speakerID, { label: speakerText.replace(/\s*#\S+\s*$/, '').trim(), optionalCode, resolvedSpeakerID });

    if (!optionalCode) {
      warnings.push(`Speaker ${speakerID} is missing an optional #speaker code.`);
    }
  }

  return speakers;
}

function buildSpeakerPersonEntities(speakerMap) {
  const entities = [];

  for (const [speakerID, details] of speakerMap.entries()) {
    const entityId = details.optionalCode ? details.optionalCode : `#${speakerID}`;
    const entity = {
      '@id': entityId,
      '@type': 'Person',
      name: details.label || speakerID,
    };

    if (details.optionalCode) {
      entity.identifier = details.optionalCode;
    }

    entities.push(entity);
  }

  return entities;
}

function buildRoCrateMetadata(collectionName, documents) {
  const crate = new ROCrate({ array: true, link: true });
  crate.addContext({ ldac: 'https://w3id.org/ldac/terms#' });
  crate.addContext({ pcdm: 'http://pcdm.org/models#' });

  crate.rootDataset['@id'] = './';
  crate.rootDataset['@type'] = ['Dataset', 'RepositoryCollection'];
  crate.rootDataset.name = collectionName;
  crate.rootDataset.conformsTo = { '@id': 'https://w3id.org/ldac/profile' };
  crate.descriptor.about = { '@id': './' };

  const collectionEntity = {
    '@id': './collection',
    '@type': 'RepositoryCollection',
    name: collectionName,
    hasMember: documents.map((document) => ({ '@id': document.objectId })),
  };

  crate.addEntity(collectionEntity);
  crate.rootDataset.hasMember = documents.map((document) => ({ '@id': document.objectId }));

  for (const document of documents) {
    const objectEntity = {
      '@id': document.objectId,
      '@type': 'RepositoryObject',
      name: document.baseName,
      hasPart: [
        { '@id': document.docxId },
        { '@id': document.csvId },
      ],
      speaker: document.speakerRefs,
    };

    const annotationEntity = {
      '@id': document.annotationId,
      '@type': 'Annotation',
      annotationOf: { '@id': document.objectId },
      annotationBody: { '@id': document.csvId },
    };

    crate.addEntity(objectEntity);
    crate.addEntity(annotationEntity);
    crate.addEntity({
      '@id': document.docxId,
      '@type': 'File',
      name: document.docxName,
      encodingFormat: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
    });
    crate.addEntity({
      '@id': document.csvId,
      '@type': 'File',
      name: document.csvName,
      encodingFormat: 'text/csv',
    });

    for (const person of document.persons) {
      crate.addEntity(person);
    }
  }

  return crate;
}

function validateSectionOrder(foundSections, warnings) {
  const expected = ['PRELIMINARIES', 'MAIN', 'POSTLIMINARIES'];
  const actual = foundSections.slice();

  if (actual.length === 0) {
    warnings.push('No section markers found. Defaulting all rows to MAIN.');
    return;
  }

  const firstMarker = actual[0];
  if (firstMarker !== 'PRELIMINARIES') {
    warnings.push(`Unexpected first section marker: ${firstMarker ?? 'none'}. Expected PRELIMINARIES.`);
  }

  const orderedSeen = [];
  for (const marker of expected) {
    if (actual.includes(marker)) {
      orderedSeen.push(marker);
    }
  }

  if (orderedSeen.length > 0 && orderedSeen[0] !== 'PRELIMINARIES') {
    warnings.push(`Section order warning: expected PRELIMINARIES before MAIN.`);
  }

  if (orderedSeen.includes('MAIN') && orderedSeen.indexOf('MAIN') < orderedSeen.indexOf('PRELIMINARIES')) {
    warnings.push('Section order warning: MAIN appears before PRELIMINARIES.');
  }

  if (actual.includes('POSTLIMINARIES')) {
    const mainIndex = actual.indexOf('MAIN');
    const postIndex = actual.indexOf('POSTLIMINARIES');
    if (mainIndex !== -1 && postIndex !== -1 && postIndex < mainIndex) {
      warnings.push('Section order warning: POSTLIMINARIES appears before MAIN.');
    }
  }
}

function parseRows(text, warnings) {
  const rows = [];
  const lines = text.split('\n');
  const speakers = parseSpeakerBlock(lines, warnings);
  let sectionOrder = [];
  let currentSection = 'MAIN';
  let transcriptStarted = false;
  let lastRow = null;

  for (const rawLine of lines) {
    const line = rawLine.trim();
    if (!line) {
      continue;
    }

    if (line === 'Speakers:') {
      transcriptStarted = false;
      continue;
    }

    if (line === 'PRELIMINARIES') {
      transcriptStarted = true;
      currentSection = 'PRE';
      sectionOrder.push(line);
      continue;
    }

    if (line === 'MAIN') {
      transcriptStarted = true;
      currentSection = 'MAIN';
      sectionOrder.push(line);
      continue;
    }

    if (line === 'POSTLIMINARIES') {
      transcriptStarted = true;
      currentSection = 'POST';
      sectionOrder.push(line);
      continue;
    }

    if (!transcriptStarted) {
      continue;
    }

    if (speakers.size > 0 && /^([A-Z][A-Z0-9]?)\s*:\s*/.test(line)) {
      const match = line.match(/^([A-Z][A-Z0-9]?)\s*:\s*(.*)$/);
      if (!match) {
        continue;
      }

      const rawSpeakerID = match[1];
      const transcriptText = match[2].trim();
      const speakerDetails = speakers.get(rawSpeakerID);
      const speakerID = speakerDetails?.optionalCode || rawSpeakerID;
      if (!speakerID || !transcriptText) {
        continue;
      }

      lastRow = { speakerID, text: transcriptText, section: currentSection };
      rows.push(lastRow);
      continue;
    }

    if (!lastRow) {
      continue;
    }

    lastRow.text = `${lastRow.text} ${line.trim()}`.trim();
  }

  validateSectionOrder(sectionOrder, warnings);

  return rows;
}

function cleanCharacterValues(value) {
  const replacements = {
    '“': '"',
    '”': '"',
    '‘': "'",
    '’': "'",
    '—': '-',
    '–': '-',
  };

  if (typeof value !== 'string') {
    return value;
  }

  let cleaned = value.trim();
  for (const [oldChar, newChar] of Object.entries(replacements)) {
    cleaned = cleaned.replaceAll(oldChar, newChar);
  }

  return cleaned;
}

function collectCharacterInventory(rows) {
  const chars = new Set();
  for (const row of rows) {
    const values = [row.speakerID, row.text, row.section];
    for (const value of values) {
      const text = String(value ?? '');
      for (const char of text) {
        chars.add(char);
      }
    }
  }

  return [...chars].sort();
}

function collectUnresolvedSpeakerRows(rows) {
  return rows
    .map((row, index) => ({ index, speakerID: row.speakerID || '' }))
    .filter(({ speakerID }) => !String(speakerID).includes('#'));
}

function formatUnresolvedSpeakerRows(rows) {
  const unresolved = collectUnresolvedSpeakerRows(rows);
  if (unresolved.length === 0) {
    return 'Unresolved speakerIDs: none';
  }

  const lines = [`Unresolved speakerIDs (${unresolved.length}):`];
  for (const item of unresolved) {
    lines.push(`Row ${item.index}: ${item.speakerID}`);
  }

  return lines.join('\n');
}

function formatCharacterInventory(rows) {
  const chars = collectCharacterInventory(rows);
  const lines = ['Character inventory:'];

  for (const char of chars) {
    const code = `U+${char.codePointAt(0).toString(16).toUpperCase().padStart(4, '0')}`;
    const name = unicodeName(char) || unicodeName(char.codePointAt(0)) || '<NO NAME>';
    lines.push(`${JSON.stringify(char)}  ${code}  ${name}`);
  }

  return lines.join('\n');
}

function applySectionRanges(rows, config) {
  return rows.map((row) => ({
    ...row,
    section: row.section || 'MAIN',
  }));
}

function toCsv(rows) {
  const output = [];
  output.push('speakerID,text,section');

  for (const row of rows) {
    const speakerID = escapeCsv(row.speakerID || '');
    const text = escapeCsv(row.text || '');
    const section = escapeCsv(row.section || 'MAIN');
    output.push(`${speakerID},${text},${section}`);
  }

  return output.join('\n') + '\n';
}

function escapeCsv(value) {
  const stringValue = String(value ?? '');
  if (/[",\n]/.test(stringValue)) {
    return `"${stringValue.replace(/"/g, '""')}"`;
  }
  return stringValue;
}

function writeLogFile(logPath, message) {
  fs.writeFileSync(logPath, `${message}\n`, 'utf8');
}

function processFile(filePath, outputDir, config) {
  const paragraphs = extractParagraphs(filePath);
  const cleanedText = normalizeText(paragraphs.join('\n'));
  const mergedText = mergeContinuationLines(cleanedText);
  const warnings = [];
  const speakerMap = parseSpeakerBlock(mergedText.split('\n'), warnings);

  let rows = parseRows(mergedText, warnings);

  if (config.headerRows > 0) {
    rows = rows.slice(config.headerRows);
  }

  if (config.footerRows > 0) {
    rows = rows.slice(0, Math.max(0, rows.length - config.footerRows));
  }

  rows = applySectionRanges(rows, config);
  rows = rows.map((row) => ({
    speakerID: cleanCharacterValues(row.speakerID),
    text: cleanCharacterValues(row.text),
    section: cleanCharacterValues(row.section || 'MAIN'),
  }));

  const fileName = path.basename(filePath, '.docx');
  const csvPath = path.join(outputDir, `${fileName}.csv`);
  const logPath = path.join(outputDir, `${fileName}.log.txt`);

  fs.writeFileSync(csvPath, toCsv(rows), 'utf8');

  const logSummary = [
    `Source file: ${filePath}`,
    `Rows processed: ${rows.length}`,
    `Header rows removed: ${config.headerRows}`,
    `Footer rows removed: ${config.footerRows}`,
    `PRE range: ${config.preStart ?? 'n/a'}-${config.preEnd ?? 'n/a'}`,
    `POST range: ${config.postStart ?? 'n/a'}-${config.postEnd ?? 'n/a'}`,
    `Warnings: ${warnings.length ? warnings.join('; ') : 'none'}`,
    'Transformations applied: text normalization, continuation repair, speaker block review, section classification, character cleanup.',
    '',
    formatUnresolvedSpeakerRows(rows),
    '',
    formatCharacterInventory(rows),
  ].join('\n');

  writeLogFile(logPath, logSummary);

  const sourceRelativePath = path.relative(outputDir, filePath).replace(/\\/g, '/');
  const speakerRefs = Array.from(speakerMap.entries()).map(([speakerID, details]) => ({
    '@id': details.optionalCode || `#${speakerID}`,
  }));
  const personEntities = buildSpeakerPersonEntities(speakerMap);

  if (warnings.length > 0) {
    console.warn(`Processed ${fileName}.docx with warnings: ${warnings.join('; ')}`);
  } else {
    console.log(`Processed ${fileName}.docx -> ${csvPath}`);
  }

  return {
    baseName: fileName,
    docxName: path.basename(filePath),
    csvName: `${fileName}.csv`,
    sourcePath: sourceRelativePath,
    objectId: `./${fileName}`,
    docxId: sourceRelativePath,
    csvId: `./${fileName}.csv`,
    annotationId: `#annotation-${fileName}`,
    speakerRefs,
    persons: personEntities,
  };
}

async function main() {
  try {
    const config = parseArgs(process.argv.slice(2));

    if (!ensurePythonDocxInstalled()) {
      console.error('python-docx is not installed. Run: python3 -m pip install -r requirements.txt');
      process.exit(1);
    }

    fs.mkdirSync(config.output, { recursive: true });

    if (!fs.existsSync(config.input)) {
      throw new Error(`Input directory does not exist: ${config.input}`);
    }

    const files = fs.readdirSync(config.input)
      .filter((entry) => entry.toLowerCase().endsWith('.docx') && !entry.startsWith('~$'))
      .map((entry) => path.join(config.input, entry))
      .sort();

    if (files.length === 0) {
      console.log(`No .docx files found in ${config.input}.`);
      return;
    }

    const documentRecords = [];
    for (const file of files) {
      const record = processFile(file, config.output, config);
      documentRecords.push(record);
    }

    const collectionName = path.basename(path.resolve(config.input));
    const crate = buildRoCrateMetadata(collectionName, documentRecords);
    const crateOutputPath = path.join(config.output, 'ro-crate-metadata.json');
    const xlsxOutputPath = path.join(config.output, 'ro-crate-metadata.xlsx');

    fs.writeFileSync(crateOutputPath, JSON.stringify(crate.getJson(), null, 2) + '\n', 'utf8');

    const workbook = new Workbook({ crate });
    await workbook.crateToWorkbook();
    const xlsxBytes = await workbook.workbook.xlsx.writeBuffer();
    fs.writeFileSync(xlsxOutputPath, Buffer.from(xlsxBytes));

    console.log(`RO-Crate metadata written to ${crateOutputPath}`);
    console.log(`RO-Crate workbook written to ${xlsxOutputPath}`);
  } catch (error) {
    console.error(`Error: ${error.message}`);
    process.exit(1);
  }
}

main();