adding setSpec

kill8080.sh

+TSKILL $(netstat -aon | gawk '$2~/:8080/ {  print $5 }' | head -n1 | sed 's/\r$//')

src/main/java/org/gesis/dda/filter/impl/Fqs2SsoarBundleFilter.java

@@ -32,7 +32,7 @@ public class Fqs2SsoarBundleFilter implements BundleFilter {
                     result = true;
                 }
                 else {
-                    LOG.info("filtering away bundle.reference={}", bundle.getReference() );
+                    LOG.info("Fqs2SsoarBundleFilter - filtering away bundle.reference={}", bundle.getReference() );
                     result = false;
                 }
             }

src/main/java/org/gesis/dda/filter/impl/Wzb2SsoarBundleFilter.java

@@ -10,7 +10,7 @@ import org.slf4j.LoggerFactory;
 
 public class Wzb2SsoarBundleFilter implements BundleFilter {
     
-    private final static Logger LOG = LoggerFactory.getLogger(Fqs2SsoarBundleFilter.class);
+    private final static Logger LOG = LoggerFactory.getLogger(Wzb2SsoarBundleFilter.class);
     
     /**
      * SSOAR already has everything up to and including dc.date.issued=2015
@@ -32,7 +32,7 @@ public class Wzb2SsoarBundleFilter implements BundleFilter {
                     result = true;
                 }
                 else {
-                    LOG.info("filtering away bundle.reference={}", bundle.getReference() );
+                    LOG.info("WzbSsoarBundleFilter - filtering away bundle.reference={} , wrong year issued", bundle.getReference() );
                     result = false;
                 }
             }

src/main/java/org/gesis/dda/publishing/domain/BundlesStreamSourceFactory.java

@@ -56,13 +56,14 @@ public class BundlesStreamSourceFactory {
 
     private String constructLoggingString(KmhopbssDto dtoDeserialized) {
         Object[] params = new Object[]{dtoDeserialized.getOaiPmhEndpoint(),
+                dtoDeserialized.getSetSpec(),
                 dtoDeserialized.isSecondGranularity(),
                 dtoDeserialized.getDayFrom(),
                 dtoDeserialized.getDayUntil(),
                 dtoDeserialized.getSecondFrom(),
                 dtoDeserialized.getSecondUntil()};
         
-        String msg = MessageFormat.format("endpoint={0}, isSecondGranularity={1}, dayFrom={2}, dayUntil={3}, secondFrom={4}, secondUntil={5}", params);
+        String msg = MessageFormat.format("endpoint={0}, setSpec={1}, isSecondGranularity={2}, dayFrom={3}, dayUntil={4}, secondFrom={5}, secondUntil={6} ", params);
         return msg;
     }
     
@@ -75,6 +76,7 @@ public class BundlesStreamSourceFactory {
                 log.debug("constructing an instant-based bss with {}", constructLoggingString(dtoDeserialized) );
                 bundlesStreamSource = new KnownMetadataPrefixesOaiPmhBundlesStreamSource(
                         dtoDeserialized.getOaiPmhEndpoint(),
+                        dtoDeserialized.getSetSpec(),
                         dtoDeserialized.getSecondFrom(),
                         dtoDeserialized.getSecondUntil()
                         );
@@ -83,6 +85,7 @@ public class BundlesStreamSourceFactory {
                 log.debug("constructing an localDate-based bss with {}", constructLoggingString(dtoDeserialized) );
                 bundlesStreamSource = new KnownMetadataPrefixesOaiPmhBundlesStreamSource(
                         dtoDeserialized.getOaiPmhEndpoint(),
+                        dtoDeserialized.getSetSpec(),
                         dtoDeserialized.getDayFrom(),
                         dtoDeserialized.getDayUntil()
                         );
@@ -117,8 +120,9 @@ public class BundlesStreamSourceFactory {
         else if (BundlesSourceType.XSLT_OAI_PMH == bundlesSourceType) {
             XsltOaiPmhDto dto = PersistableHelper.instantiate(data, XsltOaiPmhDto.class);
             String url = dto.getOaiPmhEndpoint();
+            String setSpec = dto.getSetSpec();
             Map<String, String> map = dto.getMetadataPrefix2XsltMap();
-            result = new XsltTransformerOaiPmhBundlesStreamSource(url, map, dto.getFrom(), dto.getUntil());
+            result = new XsltTransformerOaiPmhBundlesStreamSource(url, setSpec, map, dto.getFrom(), dto.getUntil());
         }
         else if (BundlesSourceType.XML_XSLT_SOURCE == bundlesSourceType) {
             XmlXsltDto dto = PersistableHelper.instantiate(data, XmlXsltDto.class);

src/main/java/org/gesis/dda/publishing/domain/impl/KmhopbssDto.java

@@ -11,6 +11,7 @@ import java.time.LocalDate;
 public class KmhopbssDto {
     
     private String oaiPmhEndpoint;
+    private String setSpec;
     private Instant secondFrom;
     private Instant secondUntil;
     private LocalDate dayFrom;
@@ -19,6 +20,7 @@ public class KmhopbssDto {
     
     public KmhopbssDto(String oaiPmhEndpoint, Instant secondFrom, Instant secondUntil) {
         this.oaiPmhEndpoint = oaiPmhEndpoint;
+        this.setSpec = "";
         this.secondFrom = secondFrom;
         this.secondUntil = secondUntil;
         this.isSecondGranularity = true;
@@ -26,6 +28,23 @@ public class KmhopbssDto {
     
     public KmhopbssDto(String oaiPmhEndpoint, LocalDate dayFrom, LocalDate dayUntil) {
         this.oaiPmhEndpoint = oaiPmhEndpoint;
+        this.setSpec = "";
+        this.dayFrom = dayFrom;
+        this.dayUntil = dayUntil;
+        this.isSecondGranularity = false;
+    }
+    
+    public KmhopbssDto(String oaiPmhEndpoint, String setSpec, Instant secondFrom, Instant secondUntil) {
+        this.oaiPmhEndpoint = oaiPmhEndpoint;
+        this.setSpec = setSpec;
+        this.secondFrom = secondFrom;
+        this.secondUntil = secondUntil;
+        this.isSecondGranularity = true;
+    }
+    
+    public KmhopbssDto(String oaiPmhEndpoint, String setSpec, LocalDate dayFrom, LocalDate dayUntil) {
+        this.oaiPmhEndpoint = oaiPmhEndpoint;
+        this.setSpec = setSpec;
         this.dayFrom = dayFrom;
         this.dayUntil = dayUntil;
         this.isSecondGranularity = false;
@@ -35,6 +54,10 @@ public class KmhopbssDto {
         return oaiPmhEndpoint;
     }
     
+    public String getSetSpec() {
+        return setSpec;
+    }
+    
     public Instant getSecondFrom() {
         return secondFrom;
     }

src/main/java/org/gesis/dda/publishing/domain/impl/KnownMetadataPrefixesOaiPmhBundlesStreamSource.java

@@ -31,6 +31,8 @@ public class KnownMetadataPrefixesOaiPmhBundlesStreamSource implements BundlesSt
     private final static Set<String> KNOWN_METADATA_PREFIXES = Sets.newHashSet("xoai", "oai_dc", "marcxml", "marc", "nlm");
 
     private String oaiPmhEndpoint;
+    
+    private String setSpec;
 
     private OaiClient oaiClient;
     private HarvestSpecification spec;
@@ -55,6 +57,28 @@ public class KnownMetadataPrefixesOaiPmhBundlesStreamSource implements BundlesSt
         this(oaiPmhEndpoint);
         this.spec = spec;
     }
+    // and here with setSpec
+    public KnownMetadataPrefixesOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec) {
+        this.spec = new HarvestSpecificationBuilder().build();
+        this.oaiPmhEndpoint = oaiPmhEndpoint;
+        this.setSpec = setSpec;
+        oaiClient = new ApacheHttpOaiClient(oaiPmhEndpoint);
+    }
+
+    public KnownMetadataPrefixesOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, LocalDate from, LocalDate until) {
+        this(oaiPmhEndpoint, setSpec);
+        this.spec = new HarvestSpecificationBuilder().withDayIntervalFromUntil(from, until).build();
+    }
+
+    public KnownMetadataPrefixesOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, Instant from, Instant until) {
+        this(oaiPmhEndpoint, setSpec);
+        this.spec = new HarvestSpecificationBuilder().withSecondIntervalFromUntil(from, until).build();
+    }
+
+    public KnownMetadataPrefixesOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, HarvestSpecification spec) {
+        this(oaiPmhEndpoint, setSpec);
+        this.spec = spec;
+    }
 
     @Override
     public Stream<Bundle> getBundlesStream() {

src/main/java/org/gesis/dda/publishing/domain/impl/XsltOaiPmhDto.java

@@ -11,6 +11,7 @@ import java.util.Map;
 public class XsltOaiPmhDto {
     
     private String oaiPmhEndpoint;
+    private String setSpec;
     private Map<String, String> metadataPrefix2XsltMap;
     private Instant from;
     private Instant until;
@@ -22,11 +23,23 @@ public class XsltOaiPmhDto {
         this.until = until;
     }
     
+    public XsltOaiPmhDto(String oaiPmhEndpoint, String setSpec, Map<String, String> metadataPrefix2XsltMap, Instant from, Instant until) {
+        this.oaiPmhEndpoint = oaiPmhEndpoint;
+        this.setSpec = setSpec;
+        this.metadataPrefix2XsltMap = metadataPrefix2XsltMap;
+        this.from = from;
+        this.until = until;
+    }
+    
     
     public String getOaiPmhEndpoint() {
         return oaiPmhEndpoint;
     }
     
+    public String getSetSpec() {
+        return setSpec;
+    }
+    
     public Map<String, String> getMetadataPrefix2XsltMap() {
         return metadataPrefix2XsltMap;
     }

src/main/java/org/gesis/dda/publishing/domain/impl/XsltTransformerOaiPmhBundlesStreamSource.java

@@ -52,6 +52,7 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
     private final static Logger LOG = LoggerFactory.getLogger(XsltTransformerOaiPmhBundlesStreamSource.class);
     
     private String oaiPmhEndpoint;
+    private String setSpec;
     private Map<String, String> metadataPrefix2XsltMap;
     private OaiPmhClient client;
     HarvestingIntervalType intervalType;
@@ -79,6 +80,29 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
         this.secondUntil = until;
     }
     
+    // And here come the same methods but with set specification
+    public XsltTransformerOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, Map<String, String> metadataPrefix2XsltMap) {
+        this.oaiPmhEndpoint = oaiPmhEndpoint;
+        this.setSpec = setSpec;
+        this.metadataPrefix2XsltMap = metadataPrefix2XsltMap;
+        this.intervalType = HarvestingIntervalType.FULL_HARVEST;
+        client = new OaiPmhClient(oaiPmhEndpoint);
+    }
+    
+    public XsltTransformerOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, Map<String, String> metadataPrefix2XsltMap, LocalDate from, LocalDate until) {
+        this(oaiPmhEndpoint, setSpec, metadataPrefix2XsltMap);
+        this.intervalType = HarvestingIntervalType.DAY_INTERVAL_HARVEST;
+        this.dayFrom = from;
+        this.dayUntil = until;
+    }
+    
+    public XsltTransformerOaiPmhBundlesStreamSource(String oaiPmhEndpoint, String setSpec, Map<String, String> metadataPrefix2XsltMap, Instant from, Instant until) {
+        this(oaiPmhEndpoint, setSpec, metadataPrefix2XsltMap);
+        this.intervalType = HarvestingIntervalType.SECOND_INTERVAL_HARVEST;
+        this.secondFrom = from;
+        this.secondUntil = until;
+    }
+    
     @Override
     public Stream<Bundle> getBundlesStream() {
         Stream<Bundle> resultBundleStream;
@@ -110,7 +134,8 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
         Stream<OAIPMHtype> listIdentifiersResponseStream = metadataPrefixes.
                 stream().
         flatMap(mp -> {
-            return client.listIdentifiersStream(mp, from, until, null);
+            LOG.debug("filling list identifier stream with mp {}, from {}, until {}, setSpec {}", mp, from, until, setSpec);
+            return client.listIdentifiersStream(mp, from, until, setSpec);
         });
         
         Stream<String> uniqueIdentifiersStream = listIdentifiersResponseStream.flatMap( oaiPmhType -> {
@@ -213,16 +238,17 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
                 String errorCode = e.getErrorCodeLocalPart();
                 if (null != errorCode) {
                     if ("filteraway".equals(errorCode) ) {
-                        LOG.debug("filtering away oaiPmhIdentifier={}", oaiPmhIdentifier);
+                        LOG.debug("1- filtering away oaiPmhIdentifier={}", oaiPmhIdentifier);
+                        LOG.debug("Error detail: {}", e.getCause().getLocalizedMessage());
                         resultBundle = null;
                         break;
                     }
                     else {
-                        LOG.warn("fatalError. Filtering away oaiPmhIdentifier=" + oaiPmhIdentifier, e);
+                        LOG.warn("2- fatalError. Filtering away oaiPmhIdentifier=" + oaiPmhIdentifier, e);
                     }
                 }
                 else {
-                    LOG.warn("fatalError. Filtering away oaiPmhIdentifier=" + oaiPmhIdentifier, e);
+                    LOG.warn("3- fatalError. Filtering away oaiPmhIdentifier=" + oaiPmhIdentifier, e);
                 }
                 resultBundle = null;
             }
@@ -272,6 +298,10 @@ public class XsltTransformerOaiPmhBundlesStreamSource implements BundlesStreamSo
         return oaiPmhEndpoint;
     }
     
+    public String getSetSpec() {
+        return setSpec;
+    }
+    
     @Override
     public Set<Metadatum> getAllMetadata(String scopedIdentifier) {
         return getBundle(scopedIdentifier).getMetadata();

src/main/java/org/gesis/dda/wizard/service/FeederService.java

@@ -78,7 +78,7 @@ public class FeederService {
         log.info("starting feeding");
         String sourceReference = feedingContext.getSourceReference();
         log.info("sourceReference={}", sourceReference);
-
+        // setSpec Existing?
         BundlesSource bundlesSourceEntity = bundlesSourceRepository.findByReference(sourceReference);
         log.info("bundlesSourceEntity={}", bundlesSourceEntity);
         log.info("feeding context={}", feedingContext.toString());

src/main/webapp/scripts/app/account/userRepositories/publicationsSources-xsltoaipmhaddress-edit-dialog.controller.js

@@ -5,11 +5,13 @@ angular.module('ddaApp').controller('PublicationsSourcesXsltOaiPmhAddressEditDia
         function($scope, $stateParams, $uibModalInstance, entity, $state, BundlesSource, MetadataTransformer) {
       
       $scope.entity = entity;
+      $scope.setSpec= '';
       $scope.newMetadataPrefix = '';
       $scope.iso8601From = (  new Date( angular.fromJson(entity.data).from.seconds * 1000)  ).toISOString();
       
       $scope.dataDto = {
           oaiPmhEndpoint: angular.fromJson(entity.data).oaiPmhEndpoint,
+          setSpec: angular.fromJson(entity.data).setSpec,
           metadataPrefix2XsltMap: angular.fromJson(entity.data).metadataPrefix2XsltMap,
           from: angular.fromJson(entity.data).from,
           until: angular.fromJson(entity.data).until

src/main/webapp/scripts/app/account/userRepositories/publicationsSources-xsltoaipmhaddress-edit-dialog.html

@@ -11,6 +11,9 @@
           <label>Base URL
             <input type="text" name="input" ng-model="dataDto.oaiPmhEndpoint">
           </label>
+          <label>Set Specification
+            <input type="text" name="input" ng-model="dataDto.setSpec">
+          </label>
         </div>
         
         <div>

src/test/resources/xslt/wzb-oai_dc-2-xmlbundle.xslt

@@ -32,24 +32,11 @@
     <xsl:template match="*" />
     
     <xsl:template match="/">
-        <xsl:variable name="sets">
-            <xsl:for-each select="//oai:setSpec" >
-                <xsl:value-of select="text()"/>
-                <xsl:text> ; </xsl:text>
-            </xsl:for-each>
-        </xsl:variable>
-        <xsl:choose>
-            <xsl:when test="not(contains($sets, 'hdl_10419_48639 ;'))" >
-                <xsl:value-of select="error(QName('','filteraway'), 'is not in set hdl_10419_48639')" />
-            </xsl:when>
-            <xsl:otherwise>
-                <xsl:element name="bundle">
-                    <xsl:element name="metadata">
-                        <xsl:apply-templates select="oai:OAI-PMH/oai:GetRecord/oai:record/oai:metadata/oai_dc:dc" />
-                    </xsl:element>
-                </xsl:element>
-            </xsl:otherwise>
-        </xsl:choose>
+        <xsl:element name="bundle">
+            <xsl:element name="metadata">
+                <xsl:apply-templates select="oai:OAI-PMH/oai:GetRecord/oai:record/oai:metadata/oai_dc:dc" />
+            </xsl:element>
+        </xsl:element>
     </xsl:template>
     
     <xsl:template match="oai:OAI-PMH/oai:GetRecord/oai:record/oai:metadata/oai_dc:dc">