diff --git a/sru/curl-sru.sh b/sru/curl-sru.sh new file mode 100644 index 0000000000..d884de95f8 --- /dev/null +++ b/sru/curl-sru.sh @@ -0,0 +1 @@ +curl 'localhost:8180/sru?version=1.1&operation=searchRetrieve&query=dinosaur&startRecord=2&maximumRecords=3&recordSchema=dc' diff --git a/sru/curl-xsearch.sh b/sru/curl-xsearch.sh new file mode 100644 index 0000000000..f34503d05d --- /dev/null +++ b/sru/curl-xsearch.sh @@ -0,0 +1 @@ +curl 'localhost:8180/xsearch?query=kamel&format=dc&start=1' diff --git a/sru/run-local.sh b/sru/run-local.sh new file mode 100644 index 0000000000..d1536ed3f5 --- /dev/null +++ b/sru/run-local.sh @@ -0,0 +1,7 @@ +SECRET=$1 + +if [ "$SECRET" != '' ]; then + java -Xmx2G -Dxl.secret.properties=$SECRET -jar build/libs/sru.jar +else + echo Nope. +fi diff --git a/sru/src/main/java/whelk/sru/servlet/Formats.java b/sru/src/main/java/whelk/sru/servlet/Formats.java new file mode 100644 index 0000000000..54f6fbf4f9 --- /dev/null +++ b/sru/src/main/java/whelk/sru/servlet/Formats.java @@ -0,0 +1,59 @@ +package whelk.sru.servlet; + +import java.io.IOException; +import java.util.Map; +import javax.xml.transform.TransformerConfigurationException; +import javax.xml.transform.Templates; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.stream.StreamSource; + +public class Formats { + + protected Map transformers = null; + private final TransformerFactory transformerFactory = TransformerFactory.newInstance(); + + protected enum Format { + MARC_XML, + MODS, + JSON, + DC, + REF_WORKS, + UNSUPPORTED + } + + protected static final Map FORMATS = Map.of( + "marcxml", Format.MARC_XML, + "json", Format.JSON, + "mods", Format.MODS, + "ris", Format.UNSUPPORTED, + "dc", Format.DC, + "rdfdc", Format.UNSUPPORTED, + "bibtex", Format.UNSUPPORTED, + "refworks", Format.REF_WORKS, + "harvard", Format.UNSUPPORTED, + "oxford", Format.UNSUPPORTED + ); + + public record Xslt(Templates templates, String contentType) { + + } + + public Xslt loadXslt(String name, String contentType) throws IOException, TransformerConfigurationException { + var url = Thread.currentThread().getContextClassLoader().getResource(name); + assert url != null; + var xsltSource = new StreamSource(url.openStream(), url.toExternalForm()); + return new Xslt(transformerFactory.newTemplates(xsltSource), contentType); + } + + public Formats() { + try { + transformers = Map.of( + Format.MODS, loadXslt("transformers/MARC21slim2MODS3.xsl", "text/xml"), + Format.DC, loadXslt("transformers/MARC21slim2DC.xsl", "text/xml"), + Format.REF_WORKS, loadXslt("transformers/refworks.xsl", "text/plain") + ); + } catch (IOException | TransformerConfigurationException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/sru/src/main/java/whelk/sru/servlet/SruServlet.java b/sru/src/main/java/whelk/sru/servlet/SruServlet.java index bccf387884..0475ee276f 100644 --- a/sru/src/main/java/whelk/sru/servlet/SruServlet.java +++ b/sru/src/main/java/whelk/sru/servlet/SruServlet.java @@ -20,14 +20,19 @@ import whelk.sru.cql.Translation; import whelk.util.MarcExport; import whelk.util.http.WhelkHttpServlet; - import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; +import javax.xml.transform.stream.StreamSource; +import javax.xml.transform.stream.StreamResult; +import javax.xml.transform.TransformerConfigurationException; +import javax.xml.transform.TransformerException; +import javax.xml.transform.Transformer; import java.io.*; +import java.io.InputStreamReader; import java.util.*; // Test locally like so: @@ -44,6 +49,8 @@ public class SruServlet extends WhelkHttpServlet { XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance(); ResourceLookup resourceLookup; ESSettings esSettings; + private Formats formats = null; + AppParams appParams; String explain = loadResource("explain.xml"); @@ -54,11 +61,10 @@ protected void init(Whelk whelk) { converter = new JsonLD2MarcXMLConverter(whelk.getMarcFrameConverter()); resourceLookup = ResourceLookup.load(whelk); esSettings = new ESSettings(whelk); + formats = new Formats(); appParams = new AppParams(appId, whelk); - Properties marcProperties = new Properties(); marcExportProfile = new ExportProfile(marcProperties); - try { marcProperties.load(new StringReader(loadResource("websok.properties"))); } catch (IOException e) { @@ -118,8 +124,11 @@ public void doGet(HttpServletRequest req, HttpServletResponse res) throws IOExce } Map results; + String format; + try { String CqlQueryString = getParameter(parameters, "query"); + format = getParameter(parameters, "recordSchema"); String XlQueryString = Translation.translateCqlToXlQuery(CqlQueryString); // This part is a little weird @@ -139,6 +148,23 @@ public void doGet(HttpServletRequest req, HttpServletResponse res) throws IOExce return; } + Transformer transformer = null; + String recordsschema = ""; + + try { + switch (Formats.FORMATS.getOrDefault(format, Formats.Format.MARC_XML)) { + case MARC_XML -> { transformer = null; recordsschema = "marcxml-v1.1"; } + case MODS -> { transformer = formats.transformers.get(Formats.Format.MODS).templates().newTransformer(); recordsschema = "mods-v3.0"; } + case DC -> { transformer = formats.transformers.get(Formats.Format.DC).templates().newTransformer(); recordsschema = "dc-v1.1"; } + case UNSUPPORTED -> { transformer = null; recordsschema = "marcxml-v1.1"; } + } + } + catch (TransformerException e){ + logger.info(e.getMessage()); + res.sendError(400); + return; + } + // Like the pre-existing implementation, supply only up to 10 hits per query. List items = (List) results.get("items"); if (items.size() > 10) @@ -181,27 +207,44 @@ public void doGet(HttpServletRequest req, HttpServletResponse res) throws IOExce writer.writeEndElement(); // recordPacking writer.writeStartElement("recordSchema"); - writer.writeCharacters("info:srw/schema/1/marcxml-v1.1"); + writer.writeCharacters("info:srw/schema/1/"+recordsschema); writer.writeEndElement(); // recordSchema - writer.writeStartElement("recordData"); + writer.flush(); + + out.write("".getBytes("UTF-8")); + Vector marcRecords = MarcExport.compileVirtualMarcRecord(marcExportProfile, embellished, whelk, converter); ByteArrayOutputStream baos = new ByteArrayOutputStream(); MarcXmlRecordWriter stringOutput = new MarcXmlRecordWriter(baos, "UTF-8", false); for (MarcRecord mr : marcRecords) { stringOutput.writeRecord(mr); } - stringOutput.close(); - StaxUtils.copy(xmlInputFactory.createXMLStreamReader(new StringReader(baos.toString())), writer); - writer.writeEndElement(); // recordData + + if ( transformer == null ) { + StaxUtils.copy(xmlInputFactory.createXMLStreamReader(new StringReader(baos.toString())), writer); + } else { + try { + transformer.transform(new StreamSource(new StringReader(baos.toString())), new StreamResult(out)); + } + catch (TransformerException e) { + logger.info(e.getMessage()); + res.sendError(400); + return; + } + } + out.write("".getBytes("UTF-8")); + out.flush(); writer.writeEndElement(); // record + writer.flush(); } writer.writeEndElement(); // records writer.writeEndElement(); // searchRetrieveResponse writer.writeEndDocument(); + writer.flush(); writer.close(); out.flush(); out.close(); @@ -243,4 +286,4 @@ private static String loadResource(String name) { throw new RuntimeException(e); } } -} \ No newline at end of file +} diff --git a/sru/src/main/java/whelk/sru/servlet/XSearchServlet.java b/sru/src/main/java/whelk/sru/servlet/XSearchServlet.java index 4b18cad283..3627152074 100644 --- a/sru/src/main/java/whelk/sru/servlet/XSearchServlet.java +++ b/sru/src/main/java/whelk/sru/servlet/XSearchServlet.java @@ -39,9 +39,7 @@ import javax.xml.stream.XMLStreamWriter; import javax.xml.transform.Templates; import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; import java.io.ByteArrayInputStream; @@ -50,6 +48,7 @@ import java.io.OutputStream; import java.io.StringReader; import java.nio.charset.StandardCharsets; +import java.util.Iterator; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -95,7 +94,7 @@ public class XSearchServlet extends WhelkHttpServlet { private final Logger logger = LoggerFactory.getLogger(this.getClass()); private final XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance(); - private final TransformerFactory transformerFactory = TransformerFactory.newInstance(); + private Formats formats = null; private static final String appId = "https://libris.kb.se/xsearch"; @@ -103,19 +102,6 @@ public class XSearchServlet extends WhelkHttpServlet { private static final int MAX_N = 200; private static final int DEFAULT_START = 1; - private static final Map FORMATS = Map.of( - "marcxml", Format.MARC_XML, - "json", Format.JSON, - "mods", Format.MODS, - "refworks", Format.REF_WORKS, - "ris", Format.UNSUPPORTED, - "dc", Format.UNSUPPORTED, - "rdfdc", Format.UNSUPPORTED, - "bibtex", Format.UNSUPPORTED, - "harvard", Format.UNSUPPORTED, - "oxford", Format.UNSUPPORTED - ); - private static final Map ORDER = Map.of( // "rank" is default "alphabetical", SORT_KEY_BY_LANG + ".sv", @@ -124,14 +110,6 @@ public class XSearchServlet extends WhelkHttpServlet { "-chronological", "publication.year" ); - private enum Format { - MARC_XML, - MODS, - JSON, - REF_WORKS, - UNSUPPORTED, - } - // https://libris.kb.se/help/xsearch_swe.jsp?open=tech private static class Params { public static final String QUERY = "query"; @@ -169,7 +147,6 @@ private record Xslt(Templates templates, String contentType) { ResourceLookup resourceLookup; ESSettings esSettings; AppParams appParams; - Map transformers; @Override protected void init(Whelk whelk) { @@ -177,15 +154,7 @@ protected void init(Whelk whelk) { resourceLookup = ResourceLookup.load(whelk); esSettings = new ESSettings(whelk); appParams = new AppParams(appId, whelk); - - try { - transformers = Map.of( - Format.MODS, loadXslt("transformers/MARC21slim2MODS3.xsl", "text/xml"), - Format.REF_WORKS, loadXslt("transformers/refworks.xsl", "text/plain") - ); - } catch (IOException | TransformerConfigurationException e) { - throw new IllegalStateException(e); - } + formats = new Formats(); } public void doGet(HttpServletRequest req, HttpServletResponse res) throws IOException { @@ -215,17 +184,17 @@ public void doGet2(HttpServletRequest req, HttpServletResponse res) throws IOExc .orElse(DEFAULT_N); var format = getOptionalSingleNonEmpty(Params.FORMAT, parameters) - .map(f -> FORMATS.getOrDefault(f, Format.MARC_XML)) - .orElse(Format.MARC_XML); + .map(f -> Formats.FORMATS.getOrDefault(f, Formats.Format.MARC_XML)) + .orElse(Formats.Format.MARC_XML); var includeHoldings = getOptionalSingleNonEmpty(Params.HOLDINGS, parameters) .map("true"::equals).orElse(false) - && (format == Format.MARC_XML || format == Format.MODS); + && (format == Formats.Format.MARC_XML || format == Formats.Format.MODS); boolean formatLevelFull = getOptionalSingleNonEmpty(Params.FORMAT_LEVEL, parameters) .map("full"::equals).orElse(false); - if (format == Format.UNSUPPORTED) { + if (format == Formats.Format.UNSUPPORTED) { throw new InvalidQueryException("format unsupported"); // TODO } @@ -233,7 +202,7 @@ public void doGet2(HttpServletRequest req, HttpServletResponse res) throws IOExc .map(ORDER::get) .orElse(null); - String callback = format == Format.JSON + String callback = format == Formats.Format.JSON ? getOptionalSingleNonEmpty(Params.CALLBACK, parameters).orElse(null) : null; if (callback != null @@ -266,8 +235,9 @@ public void doGet2(HttpServletRequest req, HttpServletResponse res) throws IOExc switch (format) { case MARC_XML -> sendMarcXML(res, items, start, to, totalItems, includeHoldings, formatLevelFull); case JSON -> sendJson(res, items, start, to, totalItems, callback); - case MODS -> sendTransformedMarc(res, transformers.get(Format.MODS), items, start, to, totalItems, includeHoldings, formatLevelFull); - case REF_WORKS -> sendTransformedMarc(res, transformers.get(Format.REF_WORKS), items, start, to, totalItems, includeHoldings, formatLevelFull); + case MODS -> sendTransformedMarc(res, Formats.Format.MODS, items, start, to, totalItems, includeHoldings, formatLevelFull); + case DC -> sendTransformedMarc(res, Formats.Format.DC, items, start, to, totalItems, false, false); + case REF_WORKS -> sendTransformedMarc(res, Formats.Format.REF_WORKS, items, start, to, totalItems, includeHoldings, formatLevelFull); } } catch (InvalidQueryException e) { @@ -389,7 +359,7 @@ private static void copyRecord(XMLStreamReader reader, XMLStreamWriter writer) t } private void sendTransformedMarc(HttpServletResponse res, - Xslt xslt, + Formats.Format format, List> items, int from, int to, @@ -398,13 +368,13 @@ private void sendTransformedMarc(HttpServletResponse res, boolean formatLevelFull) throws IOException, XMLStreamException, TransformerException { res.setCharacterEncoding("UTF-8"); - res.setContentType(xslt.contentType); + res.setContentType(formats.transformers.get(format).contentType()); ByteArrayOutputStream o = new ByteArrayOutputStream(); writeMarcXml(o, items, from, to, totalItems, includeHoldings, formatLevelFull); ByteArrayInputStream i = new ByteArrayInputStream(o.toByteArray()); - Transformer transformer = xslt.templates.newTransformer(); + Transformer transformer = formats.transformers.get(format).templates().newTransformer(); OutputStream out = res.getOutputStream(); transformer.transform(new StreamSource(i), new StreamResult(res.getOutputStream())); @@ -412,13 +382,6 @@ private void sendTransformedMarc(HttpServletResponse res, out.close(); } - private Xslt loadXslt(String name, String contentType) throws IOException, TransformerConfigurationException { - var url = Thread.currentThread().getContextClassLoader().getResource(name); - assert url != null; - var xsltSource = new StreamSource(url.openStream(), url.toExternalForm()); - return new Xslt(transformerFactory.newTemplates(xsltSource), contentType); - } - private String expandRecord(String bibXml, Document bib, boolean includeHoldings, boolean formatLevelFull) { try { MarcRecord bibRecord = MarcXmlRecordReader.fromXml(bibXml); diff --git a/sru/src/main/resources/transformers/MARC21slim2DC.xsl b/sru/src/main/resources/transformers/MARC21slim2DC.xsl new file mode 100644 index 0000000000..74020a0c81 --- /dev/null +++ b/sru/src/main/resources/transformers/MARC21slim2DC.xsl @@ -0,0 +1,283 @@ + + + + + + + + + + + + + + + + + + + + + + + <xsl:call-template name="subfieldSelect"> + <xsl:with-param name="codes">abfghk</xsl:with-param> + </xsl:call-template> + + + + + + + + + + + + + yes + + + + yes + + + + text + cartographic + notated music + sound recording + still image + moving image + three dimensional object + software, multimedia + mixed material + + + + + + + + + + + + + ab + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + abcdq + + + + + + + + abcdq + + + + + + + + abcdq + + + + + + + + abcdq + + + + + + + + abcdq + + + + + + + + abcdq + + + + + + + + abcd + + + + + + + + abcdu + + + + + + + + ot + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + .:,;/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sru/src/main/resources/transformers/MARC21slim2MODS3.xsl b/sru/src/main/resources/transformers/MARC21slim2MODS3.xsl index c24375d2a1..98563e8585 100644 --- a/sru/src/main/resources/transformers/MARC21slim2MODS3.xsl +++ b/sru/src/main/resources/transformers/MARC21slim2MODS3.xsl @@ -1,4 +1,4 @@ - + - +