Sru with transformers

sru/curl-sru.sh

@@ -0,0 +1 @@
+curl 'localhost:8180/sru?version=1.1&operation=searchRetrieve&query=dinosaur&startRecord=2&maximumRecords=3&recordSchema=dc'

sru/curl-xsearch.sh

@@ -0,0 +1 @@
+curl 'localhost:8180/xsearch?query=kamel&format=dc&start=1'

sru/run-local.sh

@@ -0,0 +1,7 @@
+SECRET=$1
+
+if [ "$SECRET" != '' ]; then
+	java -Xmx2G -Dxl.secret.properties=$SECRET -jar build/libs/sru.jar
+else
+	echo Nope.
+fi

sru/src/main/java/whelk/sru/servlet/Formats.java

@@ -0,0 +1,59 @@
+package whelk.sru.servlet;
+
+import java.io.IOException;
+import java.util.Map;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.Templates;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.stream.StreamSource;
+
+public class Formats {
+
+   protected Map<Format, Xslt> transformers = null;
+   private final TransformerFactory transformerFactory = TransformerFactory.newInstance();
+
+   protected enum Format {
+        MARC_XML,
+        MODS,
+        JSON,
+        DC,
+        REF_WORKS,
+        UNSUPPORTED
+    }
+
+    protected static final Map<String, Format> FORMATS = Map.of(
+            "marcxml", Format.MARC_XML,
+            "json", Format.JSON,
+            "mods", Format.MODS,
+            "ris", Format.UNSUPPORTED,
+            "dc", Format.DC,
+            "rdfdc", Format.UNSUPPORTED,
+            "bibtex", Format.UNSUPPORTED,
+            "refworks", Format.REF_WORKS,
+            "harvard", Format.UNSUPPORTED,
+            "oxford", Format.UNSUPPORTED
+    );
+
+    public record Xslt(Templates templates, String contentType) {
+
+    }
+
+    public Xslt loadXslt(String name, String contentType) throws IOException, TransformerConfigurationException {
+        var url = Thread.currentThread().getContextClassLoader().getResource(name);
+        assert url != null;
+        var xsltSource = new StreamSource(url.openStream(), url.toExternalForm());
+        return new Xslt(transformerFactory.newTemplates(xsltSource), contentType);
+    }
+
+    public Formats() {
+        try {
+            transformers = Map.of(
+                    Format.MODS, loadXslt("transformers/MARC21slim2MODS3.xsl", "text/xml"),
+                    Format.DC, loadXslt("transformers/MARC21slim2DC.xsl", "text/xml"),
+                    Format.REF_WORKS, loadXslt("transformers/refworks.xsl", "text/plain")
+            );
+        } catch (IOException | TransformerConfigurationException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+}

sru/src/main/java/whelk/sru/servlet/SruServlet.java

@@ -20,14 +20,19 @@
 import whelk.sru.cql.Translation;
 import whelk.util.MarcExport;
 import whelk.util.http.WhelkHttpServlet;
-
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.xml.stream.XMLInputFactory;
 import javax.xml.stream.XMLOutputFactory;
 import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamWriter;
+import javax.xml.transform.stream.StreamSource;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.Transformer;
 import java.io.*;
+import java.io.InputStreamReader;
 import java.util.*;
 
 // Test locally like so:
@@ -44,6 +49,8 @@ public class SruServlet extends WhelkHttpServlet {
     XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance();
     ResourceLookup resourceLookup;
     ESSettings esSettings;
+    private Formats formats = null;
+
     AppParams appParams;
 
     String explain = loadResource("explain.xml");
@@ -54,11 +61,10 @@ protected void init(Whelk whelk) {
         converter = new JsonLD2MarcXMLConverter(whelk.getMarcFrameConverter());
         resourceLookup = ResourceLookup.load(whelk);
         esSettings = new ESSettings(whelk);
+      	formats = new Formats();
         appParams = new AppParams(appId, whelk);
-
         Properties marcProperties = new Properties();
         marcExportProfile = new ExportProfile(marcProperties);
-
         try {
             marcProperties.load(new StringReader(loadResource("websok.properties")));
         } catch (IOException e) {
@@ -118,8 +124,11 @@ public void doGet(HttpServletRequest req, HttpServletResponse res) throws IOExce
         }
 
         Map<String, Object> results;
+        String format;  
+
         try {
             String CqlQueryString = getParameter(parameters, "query");
+            format = getParameter(parameters, "recordSchema");
             String XlQueryString = Translation.translateCqlToXlQuery(CqlQueryString);
 
             // This part is a little weird
@@ -139,6 +148,23 @@ public void doGet(HttpServletRequest req, HttpServletResponse res) throws IOExce
             return;
         }
 
+        Transformer transformer = null;
+        String recordsschema = "";
+
+        try {
+            switch (Formats.FORMATS.getOrDefault(format, Formats.Format.MARC_XML)) {
+                case MARC_XML -> { transformer = null; recordsschema = "marcxml-v1.1"; }
+                case MODS -> { transformer = formats.transformers.get(Formats.Format.MODS).templates().newTransformer(); recordsschema = "mods-v3.0"; }
+                case DC -> { transformer = formats.transformers.get(Formats.Format.DC).templates().newTransformer(); recordsschema = "dc-v1.1"; }
+                case UNSUPPORTED -> { transformer = null; recordsschema = "marcxml-v1.1"; }
+            }
+        }
+        catch (TransformerException e){
+            logger.info(e.getMessage());
+            res.sendError(400);
+            return;
+        }
+
         // Like the pre-existing implementation, supply only up to 10 hits per query.
         List items = (List) results.get("items");
         if (items.size() > 10)
@@ -181,27 +207,44 @@ public void doGet(HttpServletRequest req, HttpServletResponse res) throws IOExce
                 writer.writeEndElement(); // recordPacking
 
                 writer.writeStartElement("recordSchema");
-                writer.writeCharacters("info:srw/schema/1/marcxml-v1.1");
+                writer.writeCharacters("info:srw/schema/1/"+recordsschema);
                 writer.writeEndElement(); // recordSchema
 
-                writer.writeStartElement("recordData");
+        	writer.flush();
+
+                out.write("<recordData>".getBytes("UTF-8"));
+
                 Vector<MarcRecord> marcRecords = MarcExport.compileVirtualMarcRecord(marcExportProfile, embellished, whelk, converter);
                 ByteArrayOutputStream baos = new ByteArrayOutputStream();
                 MarcXmlRecordWriter stringOutput = new MarcXmlRecordWriter(baos, "UTF-8", false);
                 for (MarcRecord mr : marcRecords) {
                     stringOutput.writeRecord(mr);
                 }
-                stringOutput.close();
-                StaxUtils.copy(xmlInputFactory.createXMLStreamReader(new StringReader(baos.toString())), writer);
-                writer.writeEndElement(); // recordData
+
+                if ( transformer == null ) {
+                	StaxUtils.copy(xmlInputFactory.createXMLStreamReader(new StringReader(baos.toString())), writer);
+                } else {
+                       try {
+                           transformer.transform(new StreamSource(new StringReader(baos.toString())), new StreamResult(out));
+                       }
+                       catch (TransformerException e) {
+                           logger.info(e.getMessage());
+                           res.sendError(400);
+                           return;
+                       }
+                }
+                out.write("</recordData>".getBytes("UTF-8"));
+                out.flush();
 
                 writer.writeEndElement(); // record
+	    	writer.flush();
             }
 
             writer.writeEndElement(); // records
             writer.writeEndElement(); // searchRetrieveResponse
             writer.writeEndDocument();
 
+            writer.flush();
             writer.close();
             out.flush();
             out.close();
@@ -243,4 +286,4 @@ private static String loadResource(String name) {
             throw new RuntimeException(e);
         }
     }
-}
+}

sru/src/main/java/whelk/sru/servlet/XSearchServlet.java

@@ -39,9 +39,7 @@
 import javax.xml.stream.XMLStreamWriter;
 import javax.xml.transform.Templates;
 import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerConfigurationException;
 import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
 import javax.xml.transform.stream.StreamResult;
 import javax.xml.transform.stream.StreamSource;
 import java.io.ByteArrayInputStream;
@@ -50,6 +48,7 @@
 import java.io.OutputStream;
 import java.io.StringReader;
 import java.nio.charset.StandardCharsets;
+import java.util.Iterator;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -95,27 +94,14 @@
 public class XSearchServlet extends WhelkHttpServlet {
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
     private final XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance();
-    private final TransformerFactory transformerFactory = TransformerFactory.newInstance();
+    private Formats formats = null;
 
     private static final String appId = "https://libris.kb.se/xsearch";
 
     private static final int DEFAULT_N = 10;
     private static final int MAX_N = 200;
     private static final int DEFAULT_START = 1;
 
-    private static final Map<String, Format> FORMATS = Map.of(
-            "marcxml", Format.MARC_XML,
-            "json", Format.JSON,
-            "mods", Format.MODS,
-            "refworks", Format.REF_WORKS,
-            "ris", Format.UNSUPPORTED,
-            "dc", Format.UNSUPPORTED,
-            "rdfdc", Format.UNSUPPORTED,
-            "bibtex", Format.UNSUPPORTED,
-            "harvard", Format.UNSUPPORTED,
-            "oxford", Format.UNSUPPORTED
-    );
-
     private static final Map<String, String> ORDER = Map.of(
             // "rank" is default
             "alphabetical", SORT_KEY_BY_LANG + ".sv",
@@ -124,14 +110,6 @@ public class XSearchServlet extends WhelkHttpServlet {
             "-chronological", "publication.year"
     );
 
-    private enum Format {
-        MARC_XML,
-        MODS,
-        JSON,
-        REF_WORKS,
-        UNSUPPORTED,
-    }
-
     // https://libris.kb.se/help/xsearch_swe.jsp?open=tech
     private static class Params {
         public static final String QUERY = "query";
@@ -169,23 +147,14 @@ private record Xslt(Templates templates, String contentType) {
     ResourceLookup resourceLookup;
     ESSettings esSettings;
     AppParams appParams;
-    Map<Format, Xslt> transformers;
 
     @Override
     protected void init(Whelk whelk) {
         converter = new JsonLD2MarcXMLConverter(whelk.getMarcFrameConverter());
         resourceLookup = ResourceLookup.load(whelk);
         esSettings = new ESSettings(whelk);
         appParams = new AppParams(appId, whelk);
-
-        try {
-            transformers = Map.of(
-                    Format.MODS, loadXslt("transformers/MARC21slim2MODS3.xsl", "text/xml"),
-                    Format.REF_WORKS, loadXslt("transformers/refworks.xsl", "text/plain")
-            );
-        } catch (IOException | TransformerConfigurationException e) {
-            throw new IllegalStateException(e);
-        }
+      	formats = new Formats();
     }
 
     public void doGet(HttpServletRequest req, HttpServletResponse res) throws IOException {
@@ -215,25 +184,25 @@ public void doGet2(HttpServletRequest req, HttpServletResponse res) throws IOExc
                 .orElse(DEFAULT_N);
 
         var format = getOptionalSingleNonEmpty(Params.FORMAT, parameters)
-                .map(f -> FORMATS.getOrDefault(f, Format.MARC_XML))
-                .orElse(Format.MARC_XML);
+                .map(f -> Formats.FORMATS.getOrDefault(f, Formats.Format.MARC_XML))
+                .orElse(Formats.Format.MARC_XML);
 
         var includeHoldings = getOptionalSingleNonEmpty(Params.HOLDINGS, parameters)
                 .map("true"::equals).orElse(false)
-                && (format == Format.MARC_XML || format == Format.MODS);
+                && (format == Formats.Format.MARC_XML || format == Formats.Format.MODS);
 
         boolean formatLevelFull = getOptionalSingleNonEmpty(Params.FORMAT_LEVEL, parameters)
                 .map("full"::equals).orElse(false);
 
-        if (format == Format.UNSUPPORTED) {
+        if (format == Formats.Format.UNSUPPORTED) {
             throw new InvalidQueryException("format unsupported"); // TODO
         }
 
         String sort = getOptionalSingleNonEmpty(Params.ORDER, parameters)
                 .map(ORDER::get)
                 .orElse(null);
 
-        String callback = format == Format.JSON
+        String callback = format == Formats.Format.JSON
                 ? getOptionalSingleNonEmpty(Params.CALLBACK, parameters).orElse(null)
                 : null;
         if (callback != null
@@ -266,8 +235,9 @@ public void doGet2(HttpServletRequest req, HttpServletResponse res) throws IOExc
             switch (format) {
                 case MARC_XML -> sendMarcXML(res, items, start, to, totalItems, includeHoldings, formatLevelFull);
                 case JSON -> sendJson(res, items, start, to, totalItems, callback);
-                case MODS -> sendTransformedMarc(res, transformers.get(Format.MODS), items, start, to, totalItems, includeHoldings, formatLevelFull);
-                case REF_WORKS -> sendTransformedMarc(res, transformers.get(Format.REF_WORKS), items, start, to, totalItems, includeHoldings, formatLevelFull);
+                case MODS -> sendTransformedMarc(res, Formats.Format.MODS, items, start, to, totalItems, includeHoldings, formatLevelFull);
+                case DC -> sendTransformedMarc(res, Formats.Format.DC, items, start, to, totalItems, false, false);
+                case REF_WORKS -> sendTransformedMarc(res, Formats.Format.REF_WORKS, items, start, to, totalItems, includeHoldings, formatLevelFull);
             }
 
         } catch (InvalidQueryException e) {
@@ -389,7 +359,7 @@ private static void copyRecord(XMLStreamReader reader, XMLStreamWriter writer) t
     }
 
     private void sendTransformedMarc(HttpServletResponse res,
-                             Xslt xslt,
+                             Formats.Format format,
                              List<Map<?,?>> items,
                              int from,
                              int to,
@@ -398,27 +368,20 @@ private void sendTransformedMarc(HttpServletResponse res,
                              boolean formatLevelFull) throws IOException, XMLStreamException, TransformerException {
 
         res.setCharacterEncoding("UTF-8");
-        res.setContentType(xslt.contentType);
+        res.setContentType(formats.transformers.get(format).contentType());
 
         ByteArrayOutputStream o = new ByteArrayOutputStream();
         writeMarcXml(o, items, from, to, totalItems, includeHoldings, formatLevelFull);
         ByteArrayInputStream i = new ByteArrayInputStream(o.toByteArray());
 
-        Transformer transformer = xslt.templates.newTransformer();
+        Transformer transformer = formats.transformers.get(format).templates().newTransformer();
 
         OutputStream out = res.getOutputStream();
         transformer.transform(new StreamSource(i), new StreamResult(res.getOutputStream()));
         out.flush();
         out.close();
     }
 
-    private Xslt loadXslt(String name, String contentType) throws IOException, TransformerConfigurationException {
-        var url = Thread.currentThread().getContextClassLoader().getResource(name);
-        assert url != null;
-        var xsltSource = new StreamSource(url.openStream(), url.toExternalForm());
-        return new Xslt(transformerFactory.newTemplates(xsltSource), contentType);
-    }
-
     private String expandRecord(String bibXml, Document bib, boolean includeHoldings, boolean formatLevelFull) {
         try {
             MarcRecord bibRecord = MarcXmlRecordReader.fromXml(bibXml);