Skip to content

D
dda-wizard
Commit 63f60878 by Steinberg, Jan
Options

adding setSpec

parent 807bc1a0
Showing with 115 additions and 28 deletions
kill8080.sh 0 → 100755
TSKILL $(netstat -aon | gawk '$2~/:8080/ { print $5 }' | head -n1 | sed 's/\r$//')
src/main/java/org/gesis/dda/filter/impl/Fqs2SsoarBundleFilter.java
......@@ -32,7 +32,7 @@ public class Fqs2SsoarBundleFilter implements BundleFilter {
result = true;
}
else {
LOG.info("filtering away bundle.reference={}", bundle.getReference() );
LOG.info("Fqs2SsoarBundleFilter - filtering away bundle.reference={}", bundle.getReference() );
result = false;
}
}
......
src/main/java/org/gesis/dda/filter/impl/Wzb2SsoarBundleFilter.java
......@@ -10,7 +10,7 @@ import org.slf4j.LoggerFactory;
public class Wzb2SsoarBundleFilter implements BundleFilter {
private final static Logger LOG = LoggerFactory.getLogger(Fqs2SsoarBundleFilter.class);
private final static Logger LOG = LoggerFactory.getLogger(Wzb2SsoarBundleFilter.class);
/**
* SSOAR already has everything up to and including dc.date.issued=2015
......@@ -32,7 +32,7 @@ public class Wzb2SsoarBundleFilter implements BundleFilter {
result = true;
}
else {
LOG.info("filtering away bundle.reference={}", bundle.getReference() );
LOG.info("WzbSsoarBundleFilter - filtering away bundle.reference={} , wrong year issued", bundle.getReference() );
result = false;
}
}
......
src/main/java/org/gesis/dda/publishing/domain/BundlesStreamSourceFactory.java
......@@ -56,13 +56,14 @@ public class BundlesStreamSourceFactory {
private String constructLoggingString(KmhopbssDto dtoDeserialized) {
Object[] params = new Object[]{dtoDeserialized.getOaiPmhEndpoint(),
dtoDeserialized.getSetSpec(),
dtoDeserialized.isSecondGranularity(),
dtoDeserialized.getDayFrom(),
dtoDeserialized.getDayUntil(),
dtoDeserialized.getSecondFrom(),
dtoDeserialized.getSecondUntil()};
String msg = MessageFormat.format("endpoint={0}, isSecondGranularity={1}, dayFrom={2}, dayUntil={3}, secondFrom={4}, secondUntil={5}", params);
String msg = MessageFormat.format("endpoint={0}, setSpec={1}, isSecondGranularity={2}, dayFrom={3}, dayUntil={4}, secondFrom={5}, secondUntil={6} ", params);
return msg;
}
......@@ -75,6 +76,7 @@ public class BundlesStreamSourceFactory {
log.debug("constructing an instant-based bss with {}", constructLoggingString(dtoDeserialized) );
bundlesStreamSource = new KnownMetadataPrefixesOaiPmhBundlesStreamSource(
dtoDeserialized.getOaiPmhEndpoint(),
dtoDeserialized.getSetSpec(),
dtoDeserialized.getSecondFrom(),
dtoDeserialized.getSecondUntil()
);
......@@ -83,6 +85,7 @@ public class BundlesStreamSourceFactory {
log.debug("constructing an localDate-based bss with {}", constructLoggingString(dtoDeserialized) );
bundlesStreamSource = new KnownMetadataPrefixesOaiPmhBundlesStreamSource(
dtoDeserialized.getOaiPmhEndpoint(),
dtoDeserialized.getSetSpec(),
dtoDeserialized.getDayFrom(),
dtoDeserialized.getDayUntil()
);
......@@ -117,8 +120,9 @@ public class BundlesStreamSourceFactory {
else if (BundlesSourceType.XSLT_OAI_PMH == bundlesSourceType) {
XsltOaiPmhDto dto = PersistableHelper.instantiate(data, XsltOaiPmhDto.class);
String url = dto.getOaiPmhEndpoint();
String setSpec = dto.getSetSpec();
Map<String, String> map = dto.getMetadataPrefix2XsltMap();
result = new XsltTransformerOaiPmhBundlesStreamSource(url, map, dto.getFrom(), dto.getUntil());
result = new XsltTransformerOaiPmhBundlesStreamSource(url, setSpec, map, dto.getFrom(), dto.getUntil());
}
else if (BundlesSourceType.XML_XSLT_SOURCE == bundlesSourceType) {
XmlXsltDto dto = PersistableHelper.instantiate(data, XmlXsltDto.class);
......
src/main/java/org/gesis/dda/publishing/domain/impl/KmhopbssDto.java
......@@ -11,6 +11,7 @@ import java.time.LocalDate;
public class KmhopbssDto {
private String oaiPmhEndpoint;
private String setSpec;
private Instant secondFrom;
private Instant secondUntil;
private LocalDate dayFrom;
......@@ -19,6 +20,7 @@ public class KmhopbssDto {
public KmhopbssDto(String oaiPmhEndpoint, Instant secondFrom, Instant secondUntil) {
this.oaiPmhEndpoint = oaiPmhEndpoint;
this.setSpec = "";
this.secondFrom = secondFrom;
this.secondUntil = secondUntil;
this.isSecondGranularity = true;
......@@ -26,6 +28,23 @@ public class KmhopbssDto {
public KmhopbssDto(String oaiPmhEndpoint, LocalDate dayFrom, LocalDate dayUntil) {
this.oaiPmhEndpoint = oaiPmhEndpoint;
this.setSpec = "";
this.dayFrom = dayFrom;
this.dayUntil = dayUntil;
this.isSecondGranularity = false;
}
public KmhopbssDto(String oaiPmhEndpoint, String setSpec, Instant secondFrom, Instant secondUntil) {
this.oaiPmhEndpoint = oaiPmhEndpoint;
this.setSpec = setSpec;
this.secondFrom = secondFrom;
this.secondUntil = secondUntil;
this.isSecondGranularity = true;
}
public KmhopbssDto(String oaiPmhEndpoint, String setSpec, LocalDate dayFrom, LocalDate dayUntil) {
this.oaiPmhEndpoint = oaiPmhEndpoint;
this.setSpec = setSpec;
this.dayFrom = dayFrom;
this.dayUntil = dayUntil;
this.isSecondGranularity = false;
......@@ -35,6 +54,10 @@ public class KmhopbssDto {
return oaiPmhEndpoint;
}
public String getSetSpec() {
return setSpec;
}
public Instant getSecondFrom() {
return secondFrom;
}
......
src/main/java/org/gesis/dda/publishing/domain/impl/KnownMetadataPrefixesOaiPmhBundlesStreamSource.java
......@@ -31,6 +31,8 @@ public class KnownMetadataPrefixesOaiPmhBundlesStreamSource implements BundlesSt
private final static Set<String> KNOWN_METADATA_PREFIXES = Sets.newHashSet("xoai", "oai_dc", "marcxml", "marc", "nlm");
private String oaiPmhEndpoint;
private String setSpec;
private OaiClient oaiClient;
private HarvestSpecification spec;
......@@ -55,6 +57,28 @@ public class KnownMetadataPrefixesOaiPmhBundlesStreamSource implements BundlesSt
this(oaiPmhEndpoint);
this.spec = spec;
}
// and here with setSpec
public KnownMetadataPrefixesOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec) {
this.spec = new HarvestSpecificationBuilder().build();
this.oaiPmhEndpoint = oaiPmhEndpoint;
this.setSpec = setSpec;
oaiClient = new ApacheHttpOaiClient(oaiPmhEndpoint);
}
public KnownMetadataPrefixesOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, LocalDate from, LocalDate until) {
this(oaiPmhEndpoint, setSpec);
this.spec = new HarvestSpecificationBuilder().withDayIntervalFromUntil(from, until).build();
}
public KnownMetadataPrefixesOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, Instant from, Instant until) {
this(oaiPmhEndpoint, setSpec);
this.spec = new HarvestSpecificationBuilder().withSecondIntervalFromUntil(from, until).build();
}
public KnownMetadataPrefixesOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, HarvestSpecification spec) {
this(oaiPmhEndpoint, setSpec);
this.spec = spec;
}
@Override
public Stream<Bundle> getBundlesStream() {
......
src/main/java/org/gesis/dda/publishing/domain/impl/XsltOaiPmhDto.java
......@@ -11,6 +11,7 @@ import java.util.Map;
public class XsltOaiPmhDto {
private String oaiPmhEndpoint;
private String setSpec;
private Map<String, String> metadataPrefix2XsltMap;
private Instant from;
private Instant until;
......@@ -22,11 +23,23 @@ public class XsltOaiPmhDto {
this.until = until;
}
public XsltOaiPmhDto(String oaiPmhEndpoint, String setSpec, Map<String, String> metadataPrefix2XsltMap, Instant from, Instant until) {
this.oaiPmhEndpoint = oaiPmhEndpoint;
this.setSpec = setSpec;
this.metadataPrefix2XsltMap = metadataPrefix2XsltMap;
this.from = from;
this.until = until;
}
public String getOaiPmhEndpoint() {
return oaiPmhEndpoint;
}
public String getSetSpec() {
return setSpec;
}
public Map<String, String> getMetadataPrefix2XsltMap() {
return metadataPrefix2XsltMap;
}
......
src/main/java/org/gesis/dda/publishing/domain/impl/XsltTransformerOaiPmhBundlesStreamSource.java
......@@ -52,6 +52,7 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
private final static Logger LOG = LoggerFactory.getLogger(XsltTransformerOaiPmhBundlesStreamSource.class);
private String oaiPmhEndpoint;
private String setSpec;
private Map<String, String> metadataPrefix2XsltMap;
private OaiPmhClient client;
HarvestingIntervalType intervalType;
......@@ -79,6 +80,29 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
this.secondUntil = until;
}
// And here come the same methods but with set specification
public XsltTransformerOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, Map<String, String> metadataPrefix2XsltMap) {
this.oaiPmhEndpoint = oaiPmhEndpoint;
this.setSpec = setSpec;
this.metadataPrefix2XsltMap = metadataPrefix2XsltMap;
this.intervalType = HarvestingIntervalType.FULL_HARVEST;
client = new OaiPmhClient(oaiPmhEndpoint);
}
public XsltTransformerOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, Map<String, String> metadataPrefix2XsltMap, LocalDate from, LocalDate until) {
this(oaiPmhEndpoint, setSpec, metadataPrefix2XsltMap);
this.intervalType = HarvestingIntervalType.DAY_INTERVAL_HARVEST;
this.dayFrom = from;
this.dayUntil = until;
}
public XsltTransformerOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, Map<String, String> metadataPrefix2XsltMap, Instant from, Instant until) {
this(oaiPmhEndpoint, setSpec, metadataPrefix2XsltMap);
this.intervalType = HarvestingIntervalType.SECOND_INTERVAL_HARVEST;
this.secondFrom = from;
this.secondUntil = until;
}
@Override
public Stream<Bundle> getBundlesStream() {
Stream<Bundle> resultBundleStream;
......@@ -110,7 +134,8 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
Stream<OAIPMHtype> listIdentifiersResponseStream = metadataPrefixes.
stream().
flatMap(mp -> {
return client.listIdentifiersStream(mp, from, until, null);
LOG.debug("filling list identifier stream with mp {}, from {}, until {}, setSpec {}", mp, from, until, setSpec);
return client.listIdentifiersStream(mp, from, until, setSpec);
});
Stream<String> uniqueIdentifiersStream = listIdentifiersResponseStream.flatMap( oaiPmhType -> {
......@@ -213,16 +238,17 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
String errorCode = e.getErrorCodeLocalPart();
if (null != errorCode) {
if ("filteraway".equals(errorCode) ) {
LOG.debug("filtering away oaiPmhIdentifier={}", oaiPmhIdentifier);
LOG.debug("1- filtering away oaiPmhIdentifier={}", oaiPmhIdentifier);
LOG.debug("Error detail: {}", e.getCause().getLocalizedMessage());
resultBundle = null;
break;
}
else {
LOG.warn("fatalError. Filtering away oaiPmhIdentifier=" + oaiPmhIdentifier, e);
LOG.warn("2- fatalError. Filtering away oaiPmhIdentifier=" + oaiPmhIdentifier, e);
}
}
else {
LOG.warn("fatalError. Filtering away oaiPmhIdentifier=" + oaiPmhIdentifier, e);
LOG.warn("3- fatalError. Filtering away oaiPmhIdentifier=" + oaiPmhIdentifier, e);
}
resultBundle = null;
}
......@@ -272,6 +298,10 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
return oaiPmhEndpoint;
}
public String getSetSpec() {
return setSpec;
}
@Override
public Set<Metadatum> getAllMetadata(String scopedIdentifier) {
return getBundle(scopedIdentifier).getMetadata();
......
src/main/java/org/gesis/dda/wizard/service/FeederService.java
......@@ -78,7 +78,7 @@ public class FeederService {
log.info("starting feeding");
String sourceReference = feedingContext.getSourceReference();
log.info("sourceReference={}", sourceReference);
// setSpec Existing?
BundlesSource bundlesSourceEntity = bundlesSourceRepository.findByReference(sourceReference);
log.info("bundlesSourceEntity={}", bundlesSourceEntity);
log.info("feeding context={}", feedingContext.toString());
......
src/main/webapp/scripts/app/account/userRepositories/publicationsSources-xsltoaipmhaddress-edit-dialog.controller.js
......@@ -5,11 +5,13 @@ angular.module('ddaApp').controller('PublicationsSourcesXsltOaiPmhAddressEditDia
function($scope, $stateParams, $uibModalInstance, entity, $state, BundlesSource, MetadataTransformer) {
$scope.entity = entity;
$scope.setSpec= '';
$scope.newMetadataPrefix = '';
$scope.iso8601From = ( new Date( angular.fromJson(entity.data).from.seconds * 1000) ).toISOString();
$scope.dataDto = {
oaiPmhEndpoint: angular.fromJson(entity.data).oaiPmhEndpoint,
setSpec: angular.fromJson(entity.data).setSpec,
metadataPrefix2XsltMap: angular.fromJson(entity.data).metadataPrefix2XsltMap,
from: angular.fromJson(entity.data).from,
until: angular.fromJson(entity.data).until
......
src/main/webapp/scripts/app/account/userRepositories/publicationsSources-xsltoaipmhaddress-edit-dialog.html
......@@ -11,6 +11,9 @@
<label>Base URL
<input type="text" name="input" ng-model="dataDto.oaiPmhEndpoint">
</label>
<label>Set Specification
<input type="text" name="input" ng-model="dataDto.setSpec">
</label>
</div>
<div>
......
src/test/resources/xslt/wzb-oai_dc-2-xmlbundle.xslt
......@@ -32,24 +32,11 @@
<xsl:template match="*" />
<xsl:template match="/">
<xsl:variable name="sets">
<xsl:for-each select="//oai:setSpec" >
<xsl:value-of select="text()"/>
<xsl:text> ; </xsl:text>
</xsl:for-each>
</xsl:variable>
<xsl:choose>
<xsl:when test="not(contains($sets, 'hdl_10419_48639 ;'))" >
<xsl:value-of select="error(QName('','filteraway'), 'is not in set hdl_10419_48639')" />
</xsl:when>
<xsl:otherwise>
<xsl:element name="bundle">
<xsl:element name="metadata">
<xsl:apply-templates select="oai:OAI-PMH/oai:GetRecord/oai:record/oai:metadata/oai_dc:dc" />
</xsl:element>
</xsl:element>
</xsl:otherwise>
</xsl:choose>
<xsl:element name="bundle">
<xsl:element name="metadata">
<xsl:apply-templates select="oai:OAI-PMH/oai:GetRecord/oai:record/oai:metadata/oai_dc:dc" />
</xsl:element>
</xsl:element>
</xsl:template>
<xsl:template match="oai:OAI-PMH/oai:GetRecord/oai:record/oai:metadata/oai_dc:dc">
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment