Commit 63f60878 authored by Steinberg, Jan's avatar Steinberg, Jan
Browse files

adding setSpec

parent 807bc1a0
TSKILL $(netstat -aon | gawk '$2~/:8080/ { print $5 }' | head -n1 | sed 's/\r$//')
...@@ -32,7 +32,7 @@ public class Fqs2SsoarBundleFilter implements BundleFilter { ...@@ -32,7 +32,7 @@ public class Fqs2SsoarBundleFilter implements BundleFilter {
result = true; result = true;
} }
else { else {
LOG.info("filtering away bundle.reference={}", bundle.getReference() ); LOG.info("Fqs2SsoarBundleFilter - filtering away bundle.reference={}", bundle.getReference() );
result = false; result = false;
} }
} }
......
...@@ -10,7 +10,7 @@ import org.slf4j.LoggerFactory; ...@@ -10,7 +10,7 @@ import org.slf4j.LoggerFactory;
public class Wzb2SsoarBundleFilter implements BundleFilter { public class Wzb2SsoarBundleFilter implements BundleFilter {
private final static Logger LOG = LoggerFactory.getLogger(Fqs2SsoarBundleFilter.class); private final static Logger LOG = LoggerFactory.getLogger(Wzb2SsoarBundleFilter.class);
/** /**
* SSOAR already has everything up to and including dc.date.issued=2015 * SSOAR already has everything up to and including dc.date.issued=2015
...@@ -32,7 +32,7 @@ public class Wzb2SsoarBundleFilter implements BundleFilter { ...@@ -32,7 +32,7 @@ public class Wzb2SsoarBundleFilter implements BundleFilter {
result = true; result = true;
} }
else { else {
LOG.info("filtering away bundle.reference={}", bundle.getReference() ); LOG.info("WzbSsoarBundleFilter - filtering away bundle.reference={} , wrong year issued", bundle.getReference() );
result = false; result = false;
} }
} }
......
...@@ -56,13 +56,14 @@ public class BundlesStreamSourceFactory { ...@@ -56,13 +56,14 @@ public class BundlesStreamSourceFactory {
private String constructLoggingString(KmhopbssDto dtoDeserialized) { private String constructLoggingString(KmhopbssDto dtoDeserialized) {
Object[] params = new Object[]{dtoDeserialized.getOaiPmhEndpoint(), Object[] params = new Object[]{dtoDeserialized.getOaiPmhEndpoint(),
dtoDeserialized.getSetSpec(),
dtoDeserialized.isSecondGranularity(), dtoDeserialized.isSecondGranularity(),
dtoDeserialized.getDayFrom(), dtoDeserialized.getDayFrom(),
dtoDeserialized.getDayUntil(), dtoDeserialized.getDayUntil(),
dtoDeserialized.getSecondFrom(), dtoDeserialized.getSecondFrom(),
dtoDeserialized.getSecondUntil()}; dtoDeserialized.getSecondUntil()};
String msg = MessageFormat.format("endpoint={0}, isSecondGranularity={1}, dayFrom={2}, dayUntil={3}, secondFrom={4}, secondUntil={5}", params); String msg = MessageFormat.format("endpoint={0}, setSpec={1}, isSecondGranularity={2}, dayFrom={3}, dayUntil={4}, secondFrom={5}, secondUntil={6} ", params);
return msg; return msg;
} }
...@@ -75,6 +76,7 @@ public class BundlesStreamSourceFactory { ...@@ -75,6 +76,7 @@ public class BundlesStreamSourceFactory {
log.debug("constructing an instant-based bss with {}", constructLoggingString(dtoDeserialized) ); log.debug("constructing an instant-based bss with {}", constructLoggingString(dtoDeserialized) );
bundlesStreamSource = new KnownMetadataPrefixesOaiPmhBundlesStreamSource( bundlesStreamSource = new KnownMetadataPrefixesOaiPmhBundlesStreamSource(
dtoDeserialized.getOaiPmhEndpoint(), dtoDeserialized.getOaiPmhEndpoint(),
dtoDeserialized.getSetSpec(),
dtoDeserialized.getSecondFrom(), dtoDeserialized.getSecondFrom(),
dtoDeserialized.getSecondUntil() dtoDeserialized.getSecondUntil()
); );
...@@ -83,6 +85,7 @@ public class BundlesStreamSourceFactory { ...@@ -83,6 +85,7 @@ public class BundlesStreamSourceFactory {
log.debug("constructing an localDate-based bss with {}", constructLoggingString(dtoDeserialized) ); log.debug("constructing an localDate-based bss with {}", constructLoggingString(dtoDeserialized) );
bundlesStreamSource = new KnownMetadataPrefixesOaiPmhBundlesStreamSource( bundlesStreamSource = new KnownMetadataPrefixesOaiPmhBundlesStreamSource(
dtoDeserialized.getOaiPmhEndpoint(), dtoDeserialized.getOaiPmhEndpoint(),
dtoDeserialized.getSetSpec(),
dtoDeserialized.getDayFrom(), dtoDeserialized.getDayFrom(),
dtoDeserialized.getDayUntil() dtoDeserialized.getDayUntil()
); );
...@@ -117,8 +120,9 @@ public class BundlesStreamSourceFactory { ...@@ -117,8 +120,9 @@ public class BundlesStreamSourceFactory {
else if (BundlesSourceType.XSLT_OAI_PMH == bundlesSourceType) { else if (BundlesSourceType.XSLT_OAI_PMH == bundlesSourceType) {
XsltOaiPmhDto dto = PersistableHelper.instantiate(data, XsltOaiPmhDto.class); XsltOaiPmhDto dto = PersistableHelper.instantiate(data, XsltOaiPmhDto.class);
String url = dto.getOaiPmhEndpoint(); String url = dto.getOaiPmhEndpoint();
String setSpec = dto.getSetSpec();
Map<String, String> map = dto.getMetadataPrefix2XsltMap(); Map<String, String> map = dto.getMetadataPrefix2XsltMap();
result = new XsltTransformerOaiPmhBundlesStreamSource(url, map, dto.getFrom(), dto.getUntil()); result = new XsltTransformerOaiPmhBundlesStreamSource(url, setSpec, map, dto.getFrom(), dto.getUntil());
} }
else if (BundlesSourceType.XML_XSLT_SOURCE == bundlesSourceType) { else if (BundlesSourceType.XML_XSLT_SOURCE == bundlesSourceType) {
XmlXsltDto dto = PersistableHelper.instantiate(data, XmlXsltDto.class); XmlXsltDto dto = PersistableHelper.instantiate(data, XmlXsltDto.class);
......
...@@ -11,6 +11,7 @@ import java.time.LocalDate; ...@@ -11,6 +11,7 @@ import java.time.LocalDate;
public class KmhopbssDto { public class KmhopbssDto {
private String oaiPmhEndpoint; private String oaiPmhEndpoint;
private String setSpec;
private Instant secondFrom; private Instant secondFrom;
private Instant secondUntil; private Instant secondUntil;
private LocalDate dayFrom; private LocalDate dayFrom;
...@@ -19,6 +20,7 @@ public class KmhopbssDto { ...@@ -19,6 +20,7 @@ public class KmhopbssDto {
public KmhopbssDto(String oaiPmhEndpoint, Instant secondFrom, Instant secondUntil) { public KmhopbssDto(String oaiPmhEndpoint, Instant secondFrom, Instant secondUntil) {
this.oaiPmhEndpoint = oaiPmhEndpoint; this.oaiPmhEndpoint = oaiPmhEndpoint;
this.setSpec = "";
this.secondFrom = secondFrom; this.secondFrom = secondFrom;
this.secondUntil = secondUntil; this.secondUntil = secondUntil;
this.isSecondGranularity = true; this.isSecondGranularity = true;
...@@ -26,6 +28,23 @@ public class KmhopbssDto { ...@@ -26,6 +28,23 @@ public class KmhopbssDto {
public KmhopbssDto(String oaiPmhEndpoint, LocalDate dayFrom, LocalDate dayUntil) { public KmhopbssDto(String oaiPmhEndpoint, LocalDate dayFrom, LocalDate dayUntil) {
this.oaiPmhEndpoint = oaiPmhEndpoint; this.oaiPmhEndpoint = oaiPmhEndpoint;
this.setSpec = "";
this.dayFrom = dayFrom;
this.dayUntil = dayUntil;
this.isSecondGranularity = false;
}
public KmhopbssDto(String oaiPmhEndpoint, String setSpec, Instant secondFrom, Instant secondUntil) {
this.oaiPmhEndpoint = oaiPmhEndpoint;
this.setSpec = setSpec;
this.secondFrom = secondFrom;
this.secondUntil = secondUntil;
this.isSecondGranularity = true;
}
public KmhopbssDto(String oaiPmhEndpoint, String setSpec, LocalDate dayFrom, LocalDate dayUntil) {
this.oaiPmhEndpoint = oaiPmhEndpoint;
this.setSpec = setSpec;
this.dayFrom = dayFrom; this.dayFrom = dayFrom;
this.dayUntil = dayUntil; this.dayUntil = dayUntil;
this.isSecondGranularity = false; this.isSecondGranularity = false;
...@@ -35,6 +54,10 @@ public class KmhopbssDto { ...@@ -35,6 +54,10 @@ public class KmhopbssDto {
return oaiPmhEndpoint; return oaiPmhEndpoint;
} }
public String getSetSpec() {
return setSpec;
}
public Instant getSecondFrom() { public Instant getSecondFrom() {
return secondFrom; return secondFrom;
} }
......
...@@ -31,6 +31,8 @@ public class KnownMetadataPrefixesOaiPmhBundlesStreamSource implements BundlesSt ...@@ -31,6 +31,8 @@ public class KnownMetadataPrefixesOaiPmhBundlesStreamSource implements BundlesSt
private final static Set<String> KNOWN_METADATA_PREFIXES = Sets.newHashSet("xoai", "oai_dc", "marcxml", "marc", "nlm"); private final static Set<String> KNOWN_METADATA_PREFIXES = Sets.newHashSet("xoai", "oai_dc", "marcxml", "marc", "nlm");
private String oaiPmhEndpoint; private String oaiPmhEndpoint;
private String setSpec;
private OaiClient oaiClient; private OaiClient oaiClient;
private HarvestSpecification spec; private HarvestSpecification spec;
...@@ -55,6 +57,28 @@ public class KnownMetadataPrefixesOaiPmhBundlesStreamSource implements BundlesSt ...@@ -55,6 +57,28 @@ public class KnownMetadataPrefixesOaiPmhBundlesStreamSource implements BundlesSt
this(oaiPmhEndpoint); this(oaiPmhEndpoint);
this.spec = spec; this.spec = spec;
} }
// and here with setSpec
public KnownMetadataPrefixesOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec) {
this.spec = new HarvestSpecificationBuilder().build();
this.oaiPmhEndpoint = oaiPmhEndpoint;
this.setSpec = setSpec;
oaiClient = new ApacheHttpOaiClient(oaiPmhEndpoint);
}
public KnownMetadataPrefixesOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, LocalDate from, LocalDate until) {
this(oaiPmhEndpoint, setSpec);
this.spec = new HarvestSpecificationBuilder().withDayIntervalFromUntil(from, until).build();
}
public KnownMetadataPrefixesOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, Instant from, Instant until) {
this(oaiPmhEndpoint, setSpec);
this.spec = new HarvestSpecificationBuilder().withSecondIntervalFromUntil(from, until).build();
}
public KnownMetadataPrefixesOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, HarvestSpecification spec) {
this(oaiPmhEndpoint, setSpec);
this.spec = spec;
}
@Override @Override
public Stream<Bundle> getBundlesStream() { public Stream<Bundle> getBundlesStream() {
......
...@@ -11,6 +11,7 @@ import java.util.Map; ...@@ -11,6 +11,7 @@ import java.util.Map;
public class XsltOaiPmhDto { public class XsltOaiPmhDto {
private String oaiPmhEndpoint; private String oaiPmhEndpoint;
private String setSpec;
private Map<String, String> metadataPrefix2XsltMap; private Map<String, String> metadataPrefix2XsltMap;
private Instant from; private Instant from;
private Instant until; private Instant until;
...@@ -22,11 +23,23 @@ public class XsltOaiPmhDto { ...@@ -22,11 +23,23 @@ public class XsltOaiPmhDto {
this.until = until; this.until = until;
} }
public XsltOaiPmhDto(String oaiPmhEndpoint, String setSpec, Map<String, String> metadataPrefix2XsltMap, Instant from, Instant until) {
this.oaiPmhEndpoint = oaiPmhEndpoint;
this.setSpec = setSpec;
this.metadataPrefix2XsltMap = metadataPrefix2XsltMap;
this.from = from;
this.until = until;
}
public String getOaiPmhEndpoint() { public String getOaiPmhEndpoint() {
return oaiPmhEndpoint; return oaiPmhEndpoint;
} }
public String getSetSpec() {
return setSpec;
}
public Map<String, String> getMetadataPrefix2XsltMap() { public Map<String, String> getMetadataPrefix2XsltMap() {
return metadataPrefix2XsltMap; return metadataPrefix2XsltMap;
} }
......
...@@ -52,6 +52,7 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo ...@@ -52,6 +52,7 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
private final static Logger LOG = LoggerFactory.getLogger(XsltTransformerOaiPmhBundlesStreamSource.class); private final static Logger LOG = LoggerFactory.getLogger(XsltTransformerOaiPmhBundlesStreamSource.class);
private String oaiPmhEndpoint; private String oaiPmhEndpoint;
private String setSpec;
private Map<String, String> metadataPrefix2XsltMap; private Map<String, String> metadataPrefix2XsltMap;
private OaiPmhClient client; private OaiPmhClient client;
HarvestingIntervalType intervalType; HarvestingIntervalType intervalType;
...@@ -79,6 +80,29 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo ...@@ -79,6 +80,29 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
this.secondUntil = until; this.secondUntil = until;
} }
// And here come the same methods but with set specification
public XsltTransformerOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, Map<String, String> metadataPrefix2XsltMap) {
this.oaiPmhEndpoint = oaiPmhEndpoint;
this.setSpec = setSpec;
this.metadataPrefix2XsltMap = metadataPrefix2XsltMap;
this.intervalType = HarvestingIntervalType.FULL_HARVEST;
client = new OaiPmhClient(oaiPmhEndpoint);
}
public XsltTransformerOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, Map<String, String> metadataPrefix2XsltMap, LocalDate from, LocalDate until) {
this(oaiPmhEndpoint, setSpec, metadataPrefix2XsltMap);
this.intervalType = HarvestingIntervalType.DAY_INTERVAL_HARVEST;
this.dayFrom = from;
this.dayUntil = until;
}
public XsltTransformerOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, Map<String, String> metadataPrefix2XsltMap, Instant from, Instant until) {
this(oaiPmhEndpoint, setSpec, metadataPrefix2XsltMap);
this.intervalType = HarvestingIntervalType.SECOND_INTERVAL_HARVEST;
this.secondFrom = from;
this.secondUntil = until;
}
@Override @Override
public Stream<Bundle> getBundlesStream() { public Stream<Bundle> getBundlesStream() {
Stream<Bundle> resultBundleStream; Stream<Bundle> resultBundleStream;
...@@ -110,7 +134,8 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo ...@@ -110,7 +134,8 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
Stream<OAIPMHtype> listIdentifiersResponseStream = metadataPrefixes. Stream<OAIPMHtype> listIdentifiersResponseStream = metadataPrefixes.
stream(). stream().
flatMap(mp -> { flatMap(mp -> {
return client.listIdentifiersStream(mp, from, until, null); LOG.debug("filling list identifier stream with mp {}, from {}, until {}, setSpec {}", mp, from, until, setSpec);
return client.listIdentifiersStream(mp, from, until, setSpec);
}); });
Stream<String> uniqueIdentifiersStream = listIdentifiersResponseStream.flatMap( oaiPmhType -> { Stream<String> uniqueIdentifiersStream = listIdentifiersResponseStream.flatMap( oaiPmhType -> {
...@@ -213,16 +238,17 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo ...@@ -213,16 +238,17 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
String errorCode = e.getErrorCodeLocalPart(); String errorCode = e.getErrorCodeLocalPart();
if (null != errorCode) { if (null != errorCode) {
if ("filteraway".equals(errorCode) ) { if ("filteraway".equals(errorCode) ) {
LOG.debug("filtering away oaiPmhIdentifier={}", oaiPmhIdentifier); LOG.debug("1- filtering away oaiPmhIdentifier={}", oaiPmhIdentifier);
LOG.debug("Error detail: {}", e.getCause().getLocalizedMessage());
resultBundle = null; resultBundle = null;
break; break;
} }
else { else {
LOG.warn("fatalError. Filtering away oaiPmhIdentifier=" + oaiPmhIdentifier, e); LOG.warn("2- fatalError. Filtering away oaiPmhIdentifier=" + oaiPmhIdentifier, e);
} }
} }
else { else {
LOG.warn("fatalError. Filtering away oaiPmhIdentifier=" + oaiPmhIdentifier, e); LOG.warn("3- fatalError. Filtering away oaiPmhIdentifier=" + oaiPmhIdentifier, e);
} }
resultBundle = null; resultBundle = null;
} }
...@@ -272,6 +298,10 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo ...@@ -272,6 +298,10 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
return oaiPmhEndpoint; return oaiPmhEndpoint;
} }
public String getSetSpec() {
return setSpec;
}
@Override @Override
public Set<Metadatum> getAllMetadata(String scopedIdentifier) { public Set<Metadatum> getAllMetadata(String scopedIdentifier) {
return getBundle(scopedIdentifier).getMetadata(); return getBundle(scopedIdentifier).getMetadata();
......
...@@ -78,7 +78,7 @@ public class FeederService { ...@@ -78,7 +78,7 @@ public class FeederService {
log.info("starting feeding"); log.info("starting feeding");
String sourceReference = feedingContext.getSourceReference(); String sourceReference = feedingContext.getSourceReference();
log.info("sourceReference={}", sourceReference); log.info("sourceReference={}", sourceReference);
// setSpec Existing?
BundlesSource bundlesSourceEntity = bundlesSourceRepository.findByReference(sourceReference); BundlesSource bundlesSourceEntity = bundlesSourceRepository.findByReference(sourceReference);
log.info("bundlesSourceEntity={}", bundlesSourceEntity); log.info("bundlesSourceEntity={}", bundlesSourceEntity);
log.info("feeding context={}", feedingContext.toString()); log.info("feeding context={}", feedingContext.toString());
......
...@@ -5,11 +5,13 @@ angular.module('ddaApp').controller('PublicationsSourcesXsltOaiPmhAddressEditDia ...@@ -5,11 +5,13 @@ angular.module('ddaApp').controller('PublicationsSourcesXsltOaiPmhAddressEditDia
function($scope, $stateParams, $uibModalInstance, entity, $state, BundlesSource, MetadataTransformer) { function($scope, $stateParams, $uibModalInstance, entity, $state, BundlesSource, MetadataTransformer) {
$scope.entity = entity; $scope.entity = entity;
$scope.setSpec= '';
$scope.newMetadataPrefix = ''; $scope.newMetadataPrefix = '';
$scope.iso8601From = ( new Date( angular.fromJson(entity.data).from.seconds * 1000) ).toISOString(); $scope.iso8601From = ( new Date( angular.fromJson(entity.data).from.seconds * 1000) ).toISOString();
$scope.dataDto = { $scope.dataDto = {
oaiPmhEndpoint: angular.fromJson(entity.data).oaiPmhEndpoint, oaiPmhEndpoint: angular.fromJson(entity.data).oaiPmhEndpoint,
setSpec: angular.fromJson(entity.data).setSpec,
metadataPrefix2XsltMap: angular.fromJson(entity.data).metadataPrefix2XsltMap, metadataPrefix2XsltMap: angular.fromJson(entity.data).metadataPrefix2XsltMap,
from: angular.fromJson(entity.data).from, from: angular.fromJson(entity.data).from,
until: angular.fromJson(entity.data).until until: angular.fromJson(entity.data).until
......
...@@ -11,6 +11,9 @@ ...@@ -11,6 +11,9 @@
<label>Base URL <label>Base URL
<input type="text" name="input" ng-model="dataDto.oaiPmhEndpoint"> <input type="text" name="input" ng-model="dataDto.oaiPmhEndpoint">
</label> </label>
<label>Set Specification
<input type="text" name="input" ng-model="dataDto.setSpec">
</label>
</div> </div>
<div> <div>
......
...@@ -32,24 +32,11 @@ ...@@ -32,24 +32,11 @@
<xsl:template match="*" /> <xsl:template match="*" />
<xsl:template match="/"> <xsl:template match="/">
<xsl:variable name="sets"> <xsl:element name="bundle">
<xsl:for-each select="//oai:setSpec" > <xsl:element name="metadata">
<xsl:value-of select="text()"/> <xsl:apply-templates select="oai:OAI-PMH/oai:GetRecord/oai:record/oai:metadata/oai_dc:dc" />
<xsl:text> ; </xsl:text> </xsl:element>
</xsl:for-each> </xsl:element>
</xsl:variable>
<xsl:choose>
<xsl:when test="not(contains($sets, 'hdl_10419_48639 ;'))" >
<xsl:value-of select="error(QName('','filteraway'), 'is not in set hdl_10419_48639')" />
</xsl:when>
<xsl:otherwise>
<xsl:element name="bundle">
<xsl:element name="metadata">
<xsl:apply-templates select="oai:OAI-PMH/oai:GetRecord/oai:record/oai:metadata/oai_dc:dc" />
</xsl:element>
</xsl:element>
</xsl:otherwise>
</xsl:choose>
</xsl:template> </xsl:template>
<xsl:template match="oai:OAI-PMH/oai:GetRecord/oai:record/oai:metadata/oai_dc:dc"> <xsl:template match="oai:OAI-PMH/oai:GetRecord/oai:record/oai:metadata/oai_dc:dc">
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment