From 09dbc9727048dfa05f53707654dceb5d1a88c5f7 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Thu, 13 Feb 2025 17:01:24 +0100 Subject: [PATCH 01/11] solr reindexing through events --- .../authorize/ResourcePolicyServiceImpl.java | 53 ++- .../dspace/event/OAIIndexEventConsumer.java | 156 ++++++++ .../main/java/org/dspace/xoai/app/XOAI.java | 53 +++ .../DSpaceObjectRestRepository.java | 5 - .../rest/repository/ItemRestRepository.java | 6 - .../ResourcePolicyRestRepository.java | 19 - .../WorkflowItemRestRepository.java | 5 - .../app/rest/utils/SolrOAIReindexer.java | 373 ------------------ dspace/config/clarin-dspace.cfg | 8 +- 9 files changed, 263 insertions(+), 415 deletions(-) create mode 100644 dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java delete mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/SolrOAIReindexer.java diff --git a/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicyServiceImpl.java b/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicyServiceImpl.java index b32a1b89465f..048c0fec6f65 100644 --- a/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicyServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicyServiceImpl.java @@ -21,6 +21,7 @@ import org.dspace.authorize.dao.ResourcePolicyDAO; import org.dspace.authorize.service.ResourcePolicyService; import org.dspace.content.DSpaceObject; +import org.dspace.content.Item; import org.dspace.content.factory.ContentServiceFactory; import org.dspace.core.Constants; import org.dspace.core.Context; @@ -28,6 +29,7 @@ import org.dspace.eperson.EPerson; import org.dspace.eperson.Group; import org.dspace.eperson.service.GroupService; +import org.dspace.event.Event; import org.springframework.beans.factory.annotation.Autowired; /** @@ -55,8 +57,6 @@ public class ResourcePolicyServiceImpl implements ResourcePolicyService { @Autowired ProvenanceService provenanceService; - @Autowired - ResourcePolicyService resourcePolicyService; protected ResourcePolicyServiceImpl() { } @@ -146,6 +146,7 @@ public List findByTypeGroupActionExceptId(Context context, DSpac public void delete(Context context, ResourcePolicy resourcePolicy) throws SQLException, AuthorizeException { // FIXME: authorizations // Remove ourself + DSpaceObject dso = resourcePolicy.getdSpaceObject(); resourcePolicyDAO.delete(context, resourcePolicy); context.turnOffAuthorisationSystem(); @@ -155,6 +156,10 @@ public void delete(Context context, ResourcePolicy resourcePolicy) throws SQLExc .updateLastModified(context, resourcePolicy.getdSpaceObject()); } context.restoreAuthSystemState(); + if (dso instanceof Item) { + Item item = (Item) dso; + context.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); + } } @@ -220,6 +225,11 @@ public ResourcePolicy clone(Context context, ResourcePolicy resourcePolicy) clone.setRpType((String) ObjectUtils.clone(resourcePolicy.getRpType())); clone.setRpDescription((String) ObjectUtils.clone(resourcePolicy.getRpDescription())); update(context, clone); + DSpaceObject dso = resourcePolicy.getdSpaceObject(); + if (dso instanceof Item) { + Item item = (Item) dso; + context.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); + } return clone; } @@ -229,6 +239,10 @@ public void removeAllPolicies(Context c, DSpaceObject o) throws SQLException, Au c.turnOffAuthorisationSystem(); contentServiceFactory.getDSpaceObjectService(o).updateLastModified(c, o); c.restoreAuthSystemState(); + if (o instanceof Item) { + Item item = (Item) o; + c.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); + } } @Override @@ -237,20 +251,26 @@ public void removePolicies(Context c, DSpaceObject o, String type) throws SQLExc c.turnOffAuthorisationSystem(); contentServiceFactory.getDSpaceObjectService(o).updateLastModified(c, o); c.restoreAuthSystemState(); + if (o instanceof Item) { + Item item = (Item) o; + c.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); + } } @Override public void removePolicies(Context c, DSpaceObject o, String type, int action) throws SQLException, AuthorizeException { // Get all read policies of the dso before removing them - List resPolicies = resourcePolicyService.find(c, o, type); - + List resPolicies = find(c, o, type); resourcePolicyDAO.deleteByDsoAndTypeAndAction(c, o, type, action); c.turnOffAuthorisationSystem(); contentServiceFactory.getDSpaceObjectService(o).updateLastModified(c, o); c.restoreAuthSystemState(); - provenanceService.removeReadPolicies(c, o, resPolicies); + if (o instanceof Item) { + Item item = (Item) o; + c.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); + } } @Override @@ -260,6 +280,10 @@ public void removeDsoGroupPolicies(Context context, DSpaceObject dso, Group grou context.turnOffAuthorisationSystem(); contentServiceFactory.getDSpaceObjectService(dso).updateLastModified(context, dso); context.restoreAuthSystemState(); + if (dso instanceof Item) { + Item item = (Item) dso; + context.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); + } } @Override @@ -279,6 +303,13 @@ public void removeAllEPersonPolicies(Context context, EPerson ePerson) throws SQ @Override public void removeGroupPolicies(Context c, Group group) throws SQLException { + List resourcePolicies = find(c, group); + for (ResourcePolicy r : resourcePolicies) { + if (r.getdSpaceObject() instanceof Item) { + Item item = (Item) r.getdSpaceObject(); + c.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); + } + } resourcePolicyDAO.deleteByGroup(c, group); } @@ -291,6 +322,10 @@ public void removePolicies(Context c, DSpaceObject o, int actionId) throws SQLEx c.turnOffAuthorisationSystem(); contentServiceFactory.getDSpaceObjectService(o).updateLastModified(c, o); c.restoreAuthSystemState(); + if (o instanceof Item) { + Item item = (Item) o; + c.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); + } } } @@ -301,6 +336,10 @@ public void removeDsoAndTypeNotEqualsToPolicies(Context c, DSpaceObject o, Strin c.turnOffAuthorisationSystem(); contentServiceFactory.getDSpaceObjectService(o).updateLastModified(c, o); c.restoreAuthSystemState(); + if (o instanceof Item) { + Item item = (Item) o; + c.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); + } } @@ -332,6 +371,10 @@ public void update(Context context, List resourcePolicies) throw // FIXME: Check authorisation resourcePolicyDAO.save(context, resourcePolicy); + if (resourcePolicy.getdSpaceObject() instanceof Item) { + Item item = (Item) resourcePolicy.getdSpaceObject(); + context.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); + } } //Update the last modified timestamp of all related DSpace Objects diff --git a/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java b/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java new file mode 100644 index 000000000000..3f7d9bf83f8f --- /dev/null +++ b/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java @@ -0,0 +1,156 @@ +package org.dspace.event; + +import org.apache.log4j.Logger; +import org.dspace.content.Bitstream; +import org.dspace.content.Bundle; +import org.dspace.content.Collection; +import org.dspace.content.Community; +import org.dspace.content.DSpaceObject; +import org.dspace.content.Item; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.ItemService; +import org.dspace.core.Constants; +import org.dspace.core.Context; +import org.dspace.utils.DSpace; +import org.dspace.xoai.app.BasicConfiguration; +import org.dspace.xoai.app.XOAI; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +import java.sql.SQLException; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +public class OAIIndexEventConsumer implements Consumer { + /** + * log4j logger + */ + private static Logger log = Logger.getLogger(OAIIndexEventConsumer.class); + + ItemService itemService = ContentServiceFactory.getInstance().getItemService(); + // collect Items, Collections, Communities that need indexing + private Set itemsToUpdate = null; + + DSpace dspace = new DSpace(); + + public void initialize() throws Exception { + + } + + /** + * Consume a content event -- just build the sets of objects to add (new) to + * the index, update, and delete. + * + * @param ctx DSpace context + * @param event Content event + */ + public void consume(Context ctx, Event event) throws Exception { + + if (itemsToUpdate == null) { + itemsToUpdate = new HashSet(); + } + + int st = event.getSubjectType(); + if (!(st == Constants.ITEM || st == Constants.BUNDLE + || st == Constants.COLLECTION || st == Constants.COMMUNITY || st == Constants.BITSTREAM)) { + log + .warn("IndexConsumer should not have been given this kind of Subject in an event, skipping: " + + event.toString()); + return; + } + + DSpaceObject subject = event.getSubject(ctx); + DSpaceObject object = event.getObject(ctx); + + + int et = event.getEventType(); + + if(object != null && event.getObjectType() == Constants.ITEM){ + //update just object + itemsToUpdate.add((Item)object); + return; + } + + if(subject != null){ + if(event.getSubjectType() == Constants.COLLECTION || event.getSubjectType() == Constants.COMMUNITY){ + if(et == Event.MODIFY || et == Event.MODIFY_METADATA || et == Event.REMOVE || et == Event.DELETE){ + //must update all the items + if(subject.getType() == Constants.COMMUNITY){ + for(Collection col : ((Community)subject).getCollections()){ + addAll(ctx, col); + } + }else{ + addAll(ctx, (Collection)subject); + } + } + }else if(event.getSubjectType() == Constants.BITSTREAM || event.getSubjectType() == Constants.BUNDLE){ + //must update owning items regardless the event + if(subject.getType() == Constants.BITSTREAM){ + for(Bundle bun : ((Bitstream)subject).getBundles()){ + itemsToUpdate.addAll(bun.getItems()); + } + } else { + itemsToUpdate.addAll(((Bundle)subject).getItems()); + } + }else if(event.getSubjectType() == Constants.ITEM){ + //any event reindex this item + itemsToUpdate.add((Item)subject); + } + } + } + + private void addAll(Context context, Collection col) throws SQLException { + Iterator i = itemService.findByCollection(context, col); + while(i.hasNext()){ + itemsToUpdate.add(i.next()); + } + } + + /** + * Process sets of objects to add, update, and delete in index. Correct for + * interactions between the sets -- e.g. objects which were deleted do not + * need to be added or updated, new objects don't also need an update, etc. + */ + public void end(Context ctx) throws Exception { + + Context anonymousContext = null; + try { + if (itemsToUpdate != null) { + + Set filtered = new HashSet(itemsToUpdate.size()); + for (Item item : itemsToUpdate) { + if (item.getHandle() == null) { + // probably submission item, skip + continue; + } + filtered.add(item); + } + + // "free" the resources + itemsToUpdate = null; + + anonymousContext = new Context(); + XOAI indexer = new XOAI(anonymousContext, false, false); + AnnotationConfigApplicationContext applicationContext = new AnnotationConfigApplicationContext( + new Class[] { BasicConfiguration.class }); + applicationContext.getAutowireCapableBeanFactory() + .autowireBean(indexer); + indexer.indexItems(filtered); + applicationContext.close(); + } + } catch (Exception e) { + itemsToUpdate = null; + throw e; + } + finally { + if(anonymousContext!=null){ + anonymousContext.complete(); + } + } + } + + public void finish(Context ctx) throws Exception { + // No-op + + } +} diff --git a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java index 764da35fbf05..5beeafcc293f 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java @@ -36,6 +36,7 @@ import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.solr.client.solrj.SolrClient; @@ -83,6 +84,9 @@ public class XOAI { private static Logger log = LogManager.getLogger(XOAI.class); + private final XOAICacheService cacheService; + private final XOAIItemCacheService itemCacheService; + // needed because the solr query only returns 10 rows by default private final Context context; private final boolean verbose; @@ -105,6 +109,14 @@ public class XOAI { private List extensionPlugins; + { + AnnotationConfigApplicationContext applicationContext = + new AnnotationConfigApplicationContext(BasicConfiguration.class); + cacheService = applicationContext.getBean(XOAICacheService.class); + itemCacheService = applicationContext.getBean(XOAIItemCacheService.class); + + } + private List getFileFormats(Item item) { List formats = new ArrayList<>(); try { @@ -719,4 +731,45 @@ private static void usage() { } } + private boolean isTest() { + try { + if (StringUtils.equals("jdbc:h2:mem:test", this.context.getDBConfig().getDatabaseUrl())) { + return true; + } + } catch (SQLException exception) { + return false; + } + + return false; + } + + /** + * Delete the item from Solr by the ID of the item + */ + private void deleteItemByQuery(Item item) throws SolrServerException, IOException { + SolrClient solrClient = solrServerResolver.getServer(); + solrClient.deleteByQuery("item.id:" + item.getID().toString()); + solrClient.commit(); + } + + public void indexItems(java.util.Collection items) throws Exception{ + for(Item item : items){ + try { + deleteItemByQuery(item); + solrServerResolver.getServer().add(this.index(item)); + } catch (IOException | XMLStreamException | SQLException | WritingXmlException | SolrServerException e) { + // Do not throw RuntimeException in tests + if (this.isTest()) { + log.error("Cannot reindex the item with ID: " + item.getID() + " because: " + e.getMessage()); + } else { + log.error("Cannot reindex the item with ID: " + item.getID() + " because: " + e.getMessage()); + throw new RuntimeException("Cannot reindex the item with ID: " + item.getID() + " because: " + + e.getMessage()); + } + } + } + solrServerResolver.getServer().commit(); + cacheService.deleteAll(); + itemCacheService.deleteAll(); + } } diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/DSpaceObjectRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/DSpaceObjectRestRepository.java index 91a505a9a645..18c18d7af14a 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/DSpaceObjectRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/DSpaceObjectRestRepository.java @@ -15,7 +15,6 @@ import org.dspace.app.rest.model.DSpaceObjectRest; import org.dspace.app.rest.model.patch.Patch; import org.dspace.app.rest.repository.patch.ResourcePatch; -import org.dspace.app.rest.utils.SolrOAIReindexer; import org.dspace.authorize.AuthorizeException; import org.dspace.content.DSpaceObject; import org.dspace.content.service.DSpaceObjectService; @@ -39,9 +38,6 @@ public abstract class DSpaceObjectRestRepository dsoService) { this.dsoService = dsoService; } @@ -67,7 +63,6 @@ protected void patchDSpaceObject(String apiCategory, String model, UUID id, Patc } resourcePatch.patch(obtainContext(), dso, patch.getOperations()); dsoService.update(obtainContext(), dso); - solrOAIReindexer.reindexItem(dso); } @Override diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ItemRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ItemRestRepository.java index e1ab6d2b2fb6..e403be71f9b3 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ItemRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ItemRestRepository.java @@ -33,7 +33,6 @@ import org.dspace.app.rest.model.ItemRest; import org.dspace.app.rest.model.patch.Patch; import org.dspace.app.rest.repository.handler.service.UriListHandlerService; -import org.dspace.app.rest.utils.SolrOAIReindexer; import org.dspace.authorize.AuthorizeException; import org.dspace.content.Bundle; import org.dspace.content.Collection; @@ -108,9 +107,6 @@ public class ItemRestRepository extends DSpaceObjectRestRepository stringL throw new RuntimeException("SQLException in " + this.getClass() + "#findBySubmitter trying to create " + "a workflow and adding it to db.", e); } - solrOAIReindexer.reindexItem(source.getItem()); //if the item go directly in published status we have to manage a status code 204 with no content if (source.getItem().isArchived()) { return null; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/SolrOAIReindexer.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/SolrOAIReindexer.java deleted file mode 100644 index c3cb1290c0db..000000000000 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/SolrOAIReindexer.java +++ /dev/null @@ -1,373 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ - -package org.dspace.app.rest.utils; - -import static com.lyncode.xoai.dataprovider.core.Granularity.Second; -import static org.dspace.xoai.util.ItemUtils.retrieveMetadata; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; -import javax.xml.stream.XMLStreamException; - -import com.lyncode.xoai.dataprovider.exceptions.WritingXmlException; -import com.lyncode.xoai.dataprovider.xml.XmlOutputContext; -import com.lyncode.xoai.dataprovider.xml.xoai.Metadata; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrInputDocument; -import org.dspace.authorize.ResourcePolicy; -import org.dspace.authorize.factory.AuthorizeServiceFactory; -import org.dspace.authorize.service.AuthorizeService; -import org.dspace.content.Bitstream; -import org.dspace.content.Bundle; -import org.dspace.content.Collection; -import org.dspace.content.Community; -import org.dspace.content.DSpaceObject; -import org.dspace.content.Item; -import org.dspace.content.MetadataField; -import org.dspace.content.MetadataValue; -import org.dspace.content.factory.ContentServiceFactory; -import org.dspace.content.service.ItemService; -import org.dspace.core.Constants; -import org.dspace.core.Context; -import org.dspace.util.SolrUtils; -import org.dspace.utils.DSpace; -import org.dspace.xoai.app.BasicConfiguration; -import org.dspace.xoai.app.XOAIExtensionItemCompilePlugin; -import org.dspace.xoai.services.api.CollectionsService; -import org.dspace.xoai.services.api.cache.XOAICacheService; -import org.dspace.xoai.services.api.cache.XOAIItemCacheService; -import org.dspace.xoai.services.api.solr.SolrServerResolver; -import org.dspace.xoai.solr.DSpaceSolrSearch; -import org.dspace.xoai.solr.exceptions.DSpaceSolrException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; -import org.springframework.stereotype.Component; - - -/** - * Serves to reindex solr oai core after item has been added, modified or deleted. - * Solr document creation (method index(Item item)) taken from XOAI.java. - * - * @author Marian Berger (marian.berger at dataquest.sk) - */ -@Component -public class SolrOAIReindexer { - - private static final Logger log = LogManager.getLogger(SolrOAIReindexer.class); - - private final XOAICacheService cacheService; - private final XOAIItemCacheService itemCacheService; - - - @Autowired - private SolrServerResolver solrServerResolver; - - @Autowired - private CollectionsService collectionsService; - - { - AnnotationConfigApplicationContext applicationContext = - new AnnotationConfigApplicationContext(BasicConfiguration.class); - cacheService = applicationContext.getBean(XOAICacheService.class); - itemCacheService = applicationContext.getBean(XOAIItemCacheService.class); - - } - - private final Context context = new Context(Context.Mode.READ_ONLY); - private final ItemService itemService = ContentServiceFactory.getInstance().getItemService(); - private final AuthorizeService authorizeService = AuthorizeServiceFactory.getInstance() - .getAuthorizeService(); - private final List extensionPlugins = new DSpace().getServiceManager() - .getServicesByType(XOAIExtensionItemCompilePlugin.class); - - private boolean isPublic(Item item) throws SQLException { - // Check if READ access allowed on this Item - return authorizeService.authorizeActionBoolean(context, item, Constants.READ); - } - - - private boolean checkIfVisibleInOAI(Item item) throws IOException { - SolrQuery params = new SolrQuery("item.id:" + item.getID().toString()).addField("item.public"); - try { - SolrDocumentList documents = DSpaceSolrSearch.query(solrServerResolver.getServer(), params); - if (documents.getNumFound() == 1) { - return (boolean) documents.get(0).getFieldValue("item.public"); - } else { - return false; - } - } catch (DSpaceSolrException | SolrServerException e) { - return false; - } - } - - private boolean checkIfIndexed(Item item) throws IOException { - SolrQuery params = new SolrQuery("item.id:" + item.getID().toString()).addField("item.id"); - try { - SolrDocumentList documents = DSpaceSolrSearch.query(solrServerResolver.getServer(), params); - return documents.getNumFound() == 1; - } catch (DSpaceSolrException | SolrServerException e) { - return false; - } - } - - private boolean willChangeStatus(Item item) throws SQLException { - List policies = authorizeService.getPoliciesActionFilter(context, item, Constants.READ); - for (ResourcePolicy policy : policies) { - if ((policy.getGroup() != null) && (policy.getGroup().getName().equals("Anonymous"))) { - if (policy.getStartDate() != null && policy.getStartDate().after(new Date())) { - return true; - } - if (policy.getEndDate() != null && policy.getEndDate().after(new Date())) { - return true; - } - } - context.uncacheEntity(policy); - } - return false; - } - - /** - * Taken from package org.dspace.xoai.app.XOAI.java together with associated methods. - */ - private SolrInputDocument index(Item item) throws SQLException, IOException, XMLStreamException, - WritingXmlException { - SolrInputDocument doc = new SolrInputDocument(); - doc.addField("item.id", item.getID().toString()); - - String handle = item.getHandle(); - doc.addField("item.handle", handle); - - boolean isEmbargoed = !isPublic(item); - boolean isCurrentlyVisible = checkIfVisibleInOAI(item); - boolean isIndexed = checkIfIndexed(item); - - /* - * If the item is not under embargo, it should be visible. If it is, - * make it invisible if this is the first time it is indexed. For - * subsequent index runs, keep the current status, so that if the item - * is embargoed again, it is flagged as deleted instead and does not - * just disappear, or if it is still under embargo, it won't become - * visible and be known to harvesters as deleted before it gets - * disseminated for the first time. The item has to be indexed directly - * after publication even if it is still embargoed, because its - * lastModified date will not change when the embargo end date (or start - * date) is reached. To circumvent this, an item which will change its - * status in the future will be marked as such. - */ - - boolean isPublic = !isEmbargoed || (isIndexed && isCurrentlyVisible); - doc.addField("item.public", isPublic); - - // if the visibility of the item will change in the future due to an - // embargo, mark it as such. - - doc.addField("item.willChangeStatus", willChangeStatus(item)); - - /* - * Mark an item as deleted not only if it is withdrawn, but also if it - * is made private, because items should not simply disappear from OAI - * with a transient deletion policy. Do not set the flag for still - * invisible embargoed items, because this will override the item.public - * flag. - */ - boolean discoverable = item.isDiscoverable(); - // The Item is not deleted when it has local metadata `local.hidden = hidden`. - // Without this, the item is not discoverable and harvestable; however, it should be harvestable via OAI-PMH. - if (!discoverable && item.isHidden()) { - discoverable = true; - } - boolean isDeleted = item.isWithdrawn() || (!discoverable) || (isEmbargoed && isPublic); - doc.addField("item.deleted", isDeleted); - - /* - * An item that is embargoed will potentially not be harvested by - * incremental harvesters if the from and until params do not encompass - * both the standard lastModified date and the anonymous-READ resource - * policy start date. The same is true for the end date, where - * harvesters might not get a tombstone record. Therefore, consider all - * relevant policy dates and the standard lastModified date and take the - * most recent of those which have already passed. - */ - doc.addField("item.lastmodified", SolrUtils.getDateFormatter() - .format(getMostRecentModificationDate(item))); - - if (item.getSubmitter() != null) { - doc.addField("item.submitter", item.getSubmitter().getEmail()); - } - - for (Collection col : item.getCollections()) { - doc.addField("item.collections", "col_" + col.getHandle().replace("/", "_")); - } - for (Community com : collectionsService.flatParentCommunities(context, item)) { - doc.addField("item.communities", "com_" + com.getHandle().replace("/", "_")); - } - - List allData = itemService.getMetadata(item, Item.ANY, Item.ANY, Item.ANY, Item.ANY); - for (MetadataValue dc : allData) { - MetadataField field = dc.getMetadataField(); - String key = "metadata." + field.getMetadataSchema().getName() + "." + field.getElement(); - if (field.getQualifier() != null) { - key += "." + field.getQualifier(); - } - doc.addField(key, dc.getValue()); - if (dc.getAuthority() != null) { - doc.addField(key + ".authority", dc.getAuthority()); - doc.addField(key + ".confidence", dc.getConfidence() + ""); - } - } - - for (String f : getFileFormats(item)) { - doc.addField("metadata.dc.format.mimetype", f); - } - - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - XmlOutputContext xmlContext = XmlOutputContext.emptyContext(out, Second); - Metadata metadata = retrieveMetadata(context, item); - - // Do any additional metadata element, depends on the plugins - for (XOAIExtensionItemCompilePlugin plugin : extensionPlugins) { - metadata = plugin.additionalMetadata(context, metadata, item); - } - - metadata.write(xmlContext); - xmlContext.getWriter().flush(); - xmlContext.getWriter().close(); - doc.addField("item.compile", out.toString()); - - return doc; - } - - private List getFileFormats(Item item) throws SQLException { - List formats = new ArrayList<>(); - - for (Bundle b : itemService.getBundles(item, "ORIGINAL")) { - for (Bitstream bs : b.getBitstreams()) { - if (!formats.contains(bs.getFormat(context).getMIMEType())) { - formats.add(bs.getFormat(context).getMIMEType()); - } - } - } - - return formats; - } - - private Date getMostRecentModificationDate(Item item) throws SQLException { - List dates = new LinkedList<>(); - List policies = authorizeService.getPoliciesActionFilter(context, item, Constants.READ); - for (ResourcePolicy policy : policies) { - if ((policy.getGroup() != null) && (policy.getGroup().getName().equals("Anonymous"))) { - if (policy.getStartDate() != null) { - dates.add(policy.getStartDate()); - } - if (policy.getEndDate() != null) { - dates.add(policy.getEndDate()); - } - } - context.uncacheEntity(policy); - } - dates.add(item.getLastModified()); - Collections.sort(dates); - Date now = new Date(); - Date lastChange = null; - for (Date d : dates) { - if (d.before(now)) { - lastChange = d; - } - } - return lastChange; - } - - public void reindexItem(DSpaceObject item) { - if (!(item instanceof Item)) { - return; - } - reindexItem((Item) item); - } - - public void reindexItem(Item item) { - if (Objects.isNull(item.getHandle())) { -// we cannot put such item into solr - return; - } - try { - // Before reindexing delete item - deleteItemByQuery(item); - SolrClient server = solrServerResolver.getServer(); - SolrInputDocument solrInput = index(item); - server.add(solrInput); - server.commit(); - cacheService.deleteAll(); - itemCacheService.deleteAll(); - } catch (IOException | XMLStreamException | SQLException | WritingXmlException | SolrServerException e) { - // Do not throw RuntimeException in tests - if (this.isTest()) { - log.error("Cannot reindex the item with ID: " + item.getID() + " because: " + e.getMessage()); - } else { - log.error("Cannot reindex the item with ID: " + item.getID() + " because: " + e.getMessage()); - throw new RuntimeException("Cannot reindex the item with ID: " + item.getID() + " because: " - + e.getMessage()); - } - } - } - - public void deleteItem(Item item) { - try { - deleteItemByQuery(item); - cacheService.deleteAll(); - itemCacheService.deleteAll(); - } catch (SolrServerException | IOException e) { - // Do not throw RuntimeException in tests - if (this.isTest()) { - log.error("Cannot reindex the Solr after deleting the item with ID: " + item.getID() + - " because: " + e.getMessage()); - } else { - log.error("Cannot reindex the Solr after deleting the item with ID: " + item.getID() + - " because: " + e.getMessage()); - throw new RuntimeException("Cannot reindex the Solr after deleting the item with ID: " + item.getID() + - " because: " + e.getMessage()); - } - - } - } - - private boolean isTest() { - try { - if (StringUtils.equals("jdbc:h2:mem:test", this.context.getDBConfig().getDatabaseUrl())) { - return true; - } - } catch (SQLException exception) { - return false; - } - - return false; - } - - /** - * Delete the item from Solr by the ID of the item - */ - private void deleteItemByQuery(Item item) throws SolrServerException, IOException { - SolrClient solrClient = solrServerResolver.getServer(); - solrClient.deleteByQuery("item.id:" + item.getID().toString()); - solrClient.commit(); - } -} diff --git a/dspace/config/clarin-dspace.cfg b/dspace/config/clarin-dspace.cfg index 53bb118d3136..7e98a5b1757d 100644 --- a/dspace/config/clarin-dspace.cfg +++ b/dspace/config/clarin-dspace.cfg @@ -226,7 +226,7 @@ event.consumer.doi.filters = Item+Modify_Metadata # Adding doi here makes DSpace send metadata updates to your doi registration agency. # Add rdf here, if you are using dspace-rdf to export your repository content as RDF. # Add iiif here, if you are using dspace-iiif. -event.dispatcher.default.consumers = versioning, discovery, eperson +event.dispatcher.default.consumers = versioning, discovery, eperson, xoai # Edit Item - Status option identifiers.item-status.register-doi = false @@ -305,4 +305,8 @@ autocomplete.custom.allowed = solr-author_ac,solr-publisher_ac,solr-dataProvider ##### METADATA EDIT ##### #### name || id ### these collections allow submitters to edit metadata of their items -#allow.edit.metadata = \ No newline at end of file +#allow.edit.metadata = + +# consumer to maintain the oai index +event.consumer.xoai.class = org.dspace.event.OAIIndexEventConsumer +event.consumer.xoai.filters = Community|Collection|Item|Bundle|Bitstream+Add|Create|Modify|Modify_Metadata|Delete|Remove \ No newline at end of file From d2e23f580e006badb18928d9ac0a875df7e51452 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Fri, 14 Feb 2025 07:41:07 +0100 Subject: [PATCH 02/11] added license header --- .../main/java/org/dspace/event/OAIIndexEventConsumer.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java b/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java index 3f7d9bf83f8f..00a5f15b10b5 100644 --- a/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java +++ b/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java @@ -1,3 +1,10 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ package org.dspace.event; import org.apache.log4j.Logger; From 173e1a32206506ed656b0a3a5d240ccae11f098b Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Fri, 14 Feb 2025 07:59:43 +0100 Subject: [PATCH 03/11] checkstyle violations, removed unused attributes --- .../dspace/event/OAIIndexEventConsumer.java | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java b/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java index 00a5f15b10b5..83efae65b2b4 100644 --- a/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java +++ b/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java @@ -38,10 +38,8 @@ public class OAIIndexEventConsumer implements Consumer { // collect Items, Collections, Communities that need indexing private Set itemsToUpdate = null; - DSpace dspace = new DSpace(); - public void initialize() throws Exception { - + // No-op } /** @@ -72,34 +70,34 @@ public void consume(Context ctx, Event event) throws Exception { int et = event.getEventType(); - if(object != null && event.getObjectType() == Constants.ITEM){ + if (object != null && event.getObjectType() == Constants.ITEM) { //update just object itemsToUpdate.add((Item)object); return; } - if(subject != null){ - if(event.getSubjectType() == Constants.COLLECTION || event.getSubjectType() == Constants.COMMUNITY){ - if(et == Event.MODIFY || et == Event.MODIFY_METADATA || et == Event.REMOVE || et == Event.DELETE){ + if (subject != null) { + if (event.getSubjectType() == Constants.COLLECTION || event.getSubjectType() == Constants.COMMUNITY) { + if (et == Event.MODIFY || et == Event.MODIFY_METADATA || et == Event.REMOVE || et == Event.DELETE) { //must update all the items - if(subject.getType() == Constants.COMMUNITY){ - for(Collection col : ((Community)subject).getCollections()){ + if (subject.getType() == Constants.COMMUNITY) { + for (Collection col : ((Community)subject).getCollections()) { addAll(ctx, col); } - }else{ + } else { addAll(ctx, (Collection)subject); } } - }else if(event.getSubjectType() == Constants.BITSTREAM || event.getSubjectType() == Constants.BUNDLE){ + } else if (event.getSubjectType() == Constants.BITSTREAM || event.getSubjectType() == Constants.BUNDLE) { //must update owning items regardless the event - if(subject.getType() == Constants.BITSTREAM){ - for(Bundle bun : ((Bitstream)subject).getBundles()){ + if (subject.getType() == Constants.BITSTREAM) { + for (Bundle bun : ((Bitstream)subject).getBundles()) { itemsToUpdate.addAll(bun.getItems()); } } else { itemsToUpdate.addAll(((Bundle)subject).getItems()); } - }else if(event.getSubjectType() == Constants.ITEM){ + } else if (event.getSubjectType() == Constants.ITEM) { //any event reindex this item itemsToUpdate.add((Item)subject); } @@ -108,7 +106,7 @@ public void consume(Context ctx, Event event) throws Exception { private void addAll(Context context, Collection col) throws SQLException { Iterator i = itemService.findByCollection(context, col); - while(i.hasNext()){ + while (i.hasNext()) { itemsToUpdate.add(i.next()); } } @@ -148,9 +146,8 @@ public void end(Context ctx) throws Exception { } catch (Exception e) { itemsToUpdate = null; throw e; - } - finally { - if(anonymousContext!=null){ + } finally { + if (anonymousContext != null) { anonymousContext.complete(); } } @@ -158,6 +155,5 @@ public void end(Context ctx) throws Exception { public void finish(Context ctx) throws Exception { // No-op - } } From d227956abece40eb9f1aaa7668020dbe9af0a064 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Fri, 14 Feb 2025 08:16:01 +0100 Subject: [PATCH 04/11] checkstyle violations --- .../java/org/dspace/event/OAIIndexEventConsumer.java | 11 +++++------ .../src/main/java/org/dspace/xoai/app/XOAI.java | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java b/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java index 83efae65b2b4..1941dd912519 100644 --- a/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java +++ b/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java @@ -7,6 +7,11 @@ */ package org.dspace.event; +import java.sql.SQLException; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + import org.apache.log4j.Logger; import org.dspace.content.Bitstream; import org.dspace.content.Bundle; @@ -18,16 +23,10 @@ import org.dspace.content.service.ItemService; import org.dspace.core.Constants; import org.dspace.core.Context; -import org.dspace.utils.DSpace; import org.dspace.xoai.app.BasicConfiguration; import org.dspace.xoai.app.XOAI; import org.springframework.context.annotation.AnnotationConfigApplicationContext; -import java.sql.SQLException; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; - public class OAIIndexEventConsumer implements Consumer { /** * log4j logger diff --git a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java index 5beeafcc293f..92ea33dd8169 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java @@ -753,7 +753,7 @@ private void deleteItemByQuery(Item item) throws SolrServerException, IOExceptio } public void indexItems(java.util.Collection items) throws Exception{ - for(Item item : items){ + for (Item item : items) { try { deleteItemByQuery(item); solrServerResolver.getServer().add(this.index(item)); From 1d953cbf455a9f4d58b6879a1151d14451dd28e9 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Fri, 14 Feb 2025 08:44:47 +0100 Subject: [PATCH 05/11] checkstyle violations --- dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java index 92ea33dd8169..1945ecb93baa 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java @@ -752,7 +752,7 @@ private void deleteItemByQuery(Item item) throws SolrServerException, IOExceptio solrClient.commit(); } - public void indexItems(java.util.Collection items) throws Exception{ + public void indexItems(java.util.Collection items) throws Exception { for (Item item : items) { try { deleteItemByQuery(item); From dd9dba4fd6638e7cbea5b960d040109411e394a4 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Fri, 14 Feb 2025 14:23:36 +0100 Subject: [PATCH 06/11] removed empty lines, if statements, doc --- .../authorize/ResourcePolicyServiceImpl.java | 4 - .../dspace/event/OAIIndexEventConsumer.java | 95 ++++++++++--------- .../main/java/org/dspace/xoai/app/XOAI.java | 1 - 3 files changed, 52 insertions(+), 48 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicyServiceImpl.java b/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicyServiceImpl.java index 048c0fec6f65..0e5c057d86fd 100644 --- a/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicyServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicyServiceImpl.java @@ -226,10 +226,6 @@ public ResourcePolicy clone(Context context, ResourcePolicy resourcePolicy) clone.setRpDescription((String) ObjectUtils.clone(resourcePolicy.getRpDescription())); update(context, clone); DSpaceObject dso = resourcePolicy.getdSpaceObject(); - if (dso instanceof Item) { - Item item = (Item) dso; - context.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); - } return clone; } diff --git a/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java b/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java index 1941dd912519..f0c8fa050867 100644 --- a/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java +++ b/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java @@ -10,6 +10,7 @@ import java.sql.SQLException; import java.util.HashSet; import java.util.Iterator; +import java.util.Objects; import java.util.Set; import org.apache.log4j.Logger; @@ -27,6 +28,11 @@ import org.dspace.xoai.app.XOAI; import org.springframework.context.annotation.AnnotationConfigApplicationContext; +/** + * The OAIIndexEventConsumer determining which items need to be indexed or updated based on the event type and subject. + * It listens for changes to items, collections, communities, bundles, and bitstreams, and updates the OAI index accordingly. + * The indexing is done using the XOAI indexer after all relevant items are collected. + */ public class OAIIndexEventConsumer implements Consumer { /** * log4j logger @@ -34,6 +40,7 @@ public class OAIIndexEventConsumer implements Consumer { private static Logger log = Logger.getLogger(OAIIndexEventConsumer.class); ItemService itemService = ContentServiceFactory.getInstance().getItemService(); + // collect Items, Collections, Communities that need indexing private Set itemsToUpdate = null; @@ -66,40 +73,41 @@ public void consume(Context ctx, Event event) throws Exception { DSpaceObject subject = event.getSubject(ctx); DSpaceObject object = event.getObject(ctx); - int et = event.getEventType(); if (object != null && event.getObjectType() == Constants.ITEM) { - //update just object + //just update the object itemsToUpdate.add((Item)object); return; } - if (subject != null) { - if (event.getSubjectType() == Constants.COLLECTION || event.getSubjectType() == Constants.COMMUNITY) { - if (et == Event.MODIFY || et == Event.MODIFY_METADATA || et == Event.REMOVE || et == Event.DELETE) { - //must update all the items - if (subject.getType() == Constants.COMMUNITY) { - for (Collection col : ((Community)subject).getCollections()) { - addAll(ctx, col); - } - } else { - addAll(ctx, (Collection)subject); - } - } - } else if (event.getSubjectType() == Constants.BITSTREAM || event.getSubjectType() == Constants.BUNDLE) { - //must update owning items regardless the event - if (subject.getType() == Constants.BITSTREAM) { - for (Bundle bun : ((Bitstream)subject).getBundles()) { - itemsToUpdate.addAll(bun.getItems()); + if (Objects.isNull(subject)) { + return; + } + + if (event.getSubjectType() == Constants.COLLECTION || event.getSubjectType() == Constants.COMMUNITY) { + if (et == Event.MODIFY || et == Event.MODIFY_METADATA || et == Event.REMOVE || et == Event.DELETE) { + //must update all the items + if (subject.getType() == Constants.COMMUNITY) { + for (Collection col : ((Community)subject).getCollections()) { + addAll(ctx, col); } } else { - itemsToUpdate.addAll(((Bundle)subject).getItems()); + addAll(ctx, (Collection)subject); } - } else if (event.getSubjectType() == Constants.ITEM) { - //any event reindex this item - itemsToUpdate.add((Item)subject); } + } else if (event.getSubjectType() == Constants.BITSTREAM || event.getSubjectType() == Constants.BUNDLE) { + //must update owning items regardless the event + if (subject.getType() == Constants.BITSTREAM) { + for (Bundle bun : ((Bitstream)subject).getBundles()) { + itemsToUpdate.addAll(bun.getItems()); + } + } else { + itemsToUpdate.addAll(((Bundle)subject).getItems()); + } + } else if (event.getSubjectType() == Constants.ITEM) { + //any event reindex this item + itemsToUpdate.add((Item)subject); } } @@ -119,29 +127,30 @@ public void end(Context ctx) throws Exception { Context anonymousContext = null; try { - if (itemsToUpdate != null) { + if (Objects.isNull(itemsToUpdate)) { + return; + } - Set filtered = new HashSet(itemsToUpdate.size()); - for (Item item : itemsToUpdate) { - if (item.getHandle() == null) { - // probably submission item, skip - continue; - } - filtered.add(item); + Set filtered = new HashSet(itemsToUpdate.size()); + for (Item item : itemsToUpdate) { + if (item.getHandle() == null) { + // probably submission item, skip + continue; } - - // "free" the resources - itemsToUpdate = null; - - anonymousContext = new Context(); - XOAI indexer = new XOAI(anonymousContext, false, false); - AnnotationConfigApplicationContext applicationContext = new AnnotationConfigApplicationContext( - new Class[] { BasicConfiguration.class }); - applicationContext.getAutowireCapableBeanFactory() - .autowireBean(indexer); - indexer.indexItems(filtered); - applicationContext.close(); + filtered.add(item); } + + // "free" the resources + itemsToUpdate = null; + + anonymousContext = new Context(); + XOAI indexer = new XOAI(anonymousContext, false, false); + AnnotationConfigApplicationContext applicationContext = new AnnotationConfigApplicationContext( + new Class[] { BasicConfiguration.class }); + applicationContext.getAutowireCapableBeanFactory() + .autowireBean(indexer); + indexer.indexItems(filtered); + applicationContext.close(); } catch (Exception e) { itemsToUpdate = null; throw e; diff --git a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java index 1945ecb93baa..59c3a6d7cadd 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java @@ -114,7 +114,6 @@ public class XOAI { new AnnotationConfigApplicationContext(BasicConfiguration.class); cacheService = applicationContext.getBean(XOAICacheService.class); itemCacheService = applicationContext.getBean(XOAIItemCacheService.class); - } private List getFileFormats(Item item) { From 247099e5e6bd76a49e8d19a8e1b4a8bea1d675c6 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Sun, 2 Mar 2025 08:55:21 +0200 Subject: [PATCH 07/11] removed isTest from v5 --- .../main/java/org/dspace/xoai/app/XOAI.java | 23 +++---------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java index 59c3a6d7cadd..deb9e5709a6a 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java @@ -730,18 +730,6 @@ private static void usage() { } } - private boolean isTest() { - try { - if (StringUtils.equals("jdbc:h2:mem:test", this.context.getDBConfig().getDatabaseUrl())) { - return true; - } - } catch (SQLException exception) { - return false; - } - - return false; - } - /** * Delete the item from Solr by the ID of the item */ @@ -757,14 +745,9 @@ public void indexItems(java.util.Collection items) throws Exception { deleteItemByQuery(item); solrServerResolver.getServer().add(this.index(item)); } catch (IOException | XMLStreamException | SQLException | WritingXmlException | SolrServerException e) { - // Do not throw RuntimeException in tests - if (this.isTest()) { - log.error("Cannot reindex the item with ID: " + item.getID() + " because: " + e.getMessage()); - } else { - log.error("Cannot reindex the item with ID: " + item.getID() + " because: " + e.getMessage()); - throw new RuntimeException("Cannot reindex the item with ID: " + item.getID() + " because: " - + e.getMessage()); - } + log.error("Cannot reindex the item with ID: " + item.getID() + " because: " + e.getMessage()); + throw new RuntimeException("Cannot reindex the item with ID: " + item.getID() + " because: " + + e.getMessage()); } } solrServerResolver.getServer().commit(); From 9a83916ea76bc0b590b939584021ca52ab72f550 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Thu, 6 Mar 2025 22:57:04 +0200 Subject: [PATCH 08/11] fix incorrect merge --- .../repository/ResourcePolicyRestRepository.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ResourcePolicyRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ResourcePolicyRestRepository.java index 5e553a807baa..c682aa4d9891 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ResourcePolicyRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ResourcePolicyRestRepository.java @@ -279,6 +279,18 @@ protected ResourcePolicyRest createAndReturn(Context context) throws AuthorizeEx } catch (SQLException excSQL) { throw new RuntimeException(excSQL.getMessage(), excSQL); } + } + + if (resourcePolicy != null) { + + resourcePolicy.setRpType(resourcePolicyRest.getPolicyType()); + resourcePolicy.setdSpaceObject(dspaceObject); + resourcePolicy.setRpName(resourcePolicyRest.getName()); + resourcePolicy.setRpDescription(resourcePolicyRest.getDescription()); + resourcePolicy.setAction(Constants.getActionID(resourcePolicyRest.getAction())); + resourcePolicy.setStartDate(resourcePolicyRest.getStartDate()); + resourcePolicy.setEndDate(resourcePolicyRest.getEndDate()); + resourcePolicyService.update(context, resourcePolicy); return converter.toRest(resourcePolicy, utils.obtainProjection()); } else { throw new UnprocessableEntityException("A resource policy must contain a valid eperson or group"); From d437b4ea953deeebc0c803ac844f58bece72cb5e Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Fri, 7 Mar 2025 09:21:25 +0200 Subject: [PATCH 09/11] fix test and checkstyle violation --- .../src/main/java/org/dspace/event/OAIIndexEventConsumer.java | 3 ++- dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java | 1 - .../app/rest/repository/ResourcePolicyRestRepository.java | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java b/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java index f0c8fa050867..76952cdc328e 100644 --- a/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java +++ b/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java @@ -30,7 +30,8 @@ /** * The OAIIndexEventConsumer determining which items need to be indexed or updated based on the event type and subject. - * It listens for changes to items, collections, communities, bundles, and bitstreams, and updates the OAI index accordingly. + * It listens for changes to items, collections, communities, + * bundles, and bitstreams, and updates the OAI index accordingly. * The indexing is done using the XOAI indexer after all relevant items are collected. */ public class OAIIndexEventConsumer implements Consumer { diff --git a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java index deb9e5709a6a..a6d82d812968 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java @@ -36,7 +36,6 @@ import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.Options; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.solr.client.solrj.SolrClient; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ResourcePolicyRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ResourcePolicyRestRepository.java index c682aa4d9891..e973476305bb 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ResourcePolicyRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ResourcePolicyRestRepository.java @@ -267,7 +267,6 @@ protected ResourcePolicyRest createAndReturn(Context context) throws AuthorizeEx } catch (SQLException excSQL) { throw new RuntimeException(excSQL.getMessage(), excSQL); } - return converter.toRest(resourcePolicy, utils.obtainProjection()); } else { try { UUID groupUuid = UUID.fromString(groupUuidStr); From bccc4fd57436b0c16e01bc1237ab8c7b5e1e3e97 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Sun, 9 Mar 2025 20:22:17 +0200 Subject: [PATCH 10/11] removed duplicities for new modify event --- .../authorize/ResourcePolicyServiceImpl.java | 53 ++++++------------- .../service/ResourcePolicyService.java | 7 +++ 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicyServiceImpl.java b/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicyServiceImpl.java index 6ea9556110db..c4074c06c5b2 100644 --- a/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicyServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicyServiceImpl.java @@ -164,10 +164,7 @@ public void delete(Context context, ResourcePolicy resourcePolicy) throws SQLExc .updateLastModified(context, resourcePolicy.getdSpaceObject()); } context.restoreAuthSystemState(); - if (dso instanceof Item) { - Item item = (Item) dso; - context.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); - } + addEventModify(context, dso); } @@ -241,10 +238,7 @@ public void removeAllPolicies(Context c, DSpaceObject o) throws SQLException, Au c.turnOffAuthorisationSystem(); contentServiceFactory.getDSpaceObjectService(o).updateLastModified(c, o); c.restoreAuthSystemState(); - if (o instanceof Item) { - Item item = (Item) o; - c.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); - } + addEventModify(c, o); } @Override @@ -253,10 +247,7 @@ public void removePolicies(Context c, DSpaceObject o, String type) throws SQLExc c.turnOffAuthorisationSystem(); contentServiceFactory.getDSpaceObjectService(o).updateLastModified(c, o); c.restoreAuthSystemState(); - if (o instanceof Item) { - Item item = (Item) o; - c.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); - } + addEventModify(c, o); } @Override @@ -269,10 +260,7 @@ public void removePolicies(Context c, DSpaceObject o, String type, int action) contentServiceFactory.getDSpaceObjectService(o).updateLastModified(c, o); c.restoreAuthSystemState(); provenanceService.removeReadPolicies(c, o, resPolicies); - if (o instanceof Item) { - Item item = (Item) o; - c.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); - } + addEventModify(c, o); } @Override @@ -282,10 +270,7 @@ public void removeDsoGroupPolicies(Context context, DSpaceObject dso, Group grou context.turnOffAuthorisationSystem(); contentServiceFactory.getDSpaceObjectService(dso).updateLastModified(context, dso); context.restoreAuthSystemState(); - if (dso instanceof Item) { - Item item = (Item) dso; - context.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); - } + addEventModify(context, dso); } @Override @@ -307,10 +292,7 @@ public void removeAllEPersonPolicies(Context context, EPerson ePerson) throws SQ public void removeGroupPolicies(Context c, Group group) throws SQLException { List resourcePolicies = find(c, group); for (ResourcePolicy r : resourcePolicies) { - if (r.getdSpaceObject() instanceof Item) { - Item item = (Item) r.getdSpaceObject(); - c.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); - } + addEventModify(c, r.getdSpaceObject()); } resourcePolicyDAO.deleteByGroup(c, group); } @@ -324,10 +306,7 @@ public void removePolicies(Context c, DSpaceObject o, int actionId) throws SQLEx c.turnOffAuthorisationSystem(); contentServiceFactory.getDSpaceObjectService(o).updateLastModified(c, o); c.restoreAuthSystemState(); - if (o instanceof Item) { - Item item = (Item) o; - c.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); - } + addEventModify(c, o); } } @@ -338,10 +317,7 @@ public void removeDsoAndTypeNotEqualsToPolicies(Context c, DSpaceObject o, Strin c.turnOffAuthorisationSystem(); contentServiceFactory.getDSpaceObjectService(o).updateLastModified(c, o); c.restoreAuthSystemState(); - if (o instanceof Item) { - Item item = (Item) o; - c.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); - } + addEventModify(c, o); } @@ -373,10 +349,7 @@ public void update(Context context, List resourcePolicies) throw // FIXME: Check authorisation resourcePolicyDAO.save(context, resourcePolicy); - if (resourcePolicy.getdSpaceObject() instanceof Item) { - Item item = (Item) resourcePolicy.getdSpaceObject(); - context.addEvent(new Event(Event.MODIFY, -1, null, Constants.ITEM, item.getID(), "")); - } + addEventModify(context, resourcePolicy.getdSpaceObject()); } //Update the last modified timestamp of all related DSpace Objects @@ -475,4 +448,12 @@ public boolean isMyResourcePolicy(Context context, EPerson eperson, Integer id) } return isMy; } + + public void addEventModify(Context context, DSpaceObject dso) { + if (dso instanceof Item) { + Item item = (Item) dso; + context.addEvent(new Event(Event.MODIFY, -1, null, + Constants.ITEM, item.getID(), "")); + } + } } diff --git a/dspace-api/src/main/java/org/dspace/authorize/service/ResourcePolicyService.java b/dspace-api/src/main/java/org/dspace/authorize/service/ResourcePolicyService.java index 523bd64006d0..fe78342f3545 100644 --- a/dspace-api/src/main/java/org/dspace/authorize/service/ResourcePolicyService.java +++ b/dspace-api/src/main/java/org/dspace/authorize/service/ResourcePolicyService.java @@ -311,4 +311,11 @@ public List findByGroupAndResourceUuid(Context context, Group gr */ public boolean isMyResourcePolicy(Context context, EPerson eperson, Integer id) throws SQLException; + /** + * Add MODIFY event when the provided DSpaceObject is an Item. + * + * @param context The DSpace context. + * @param dso The DSpaceObject. + */ + public void addEventModify(Context context, DSpaceObject dso); } From a208d99890fe317a7ad8a4541a98a6b65b961287 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Mon, 10 Mar 2025 08:27:25 +0200 Subject: [PATCH 11/11] used isNull and nonNull in OAIIndexEventConsumer --- .../main/java/org/dspace/event/OAIIndexEventConsumer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java b/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java index 76952cdc328e..7664cf2cab3f 100644 --- a/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java +++ b/dspace-oai/src/main/java/org/dspace/event/OAIIndexEventConsumer.java @@ -58,7 +58,7 @@ public void initialize() throws Exception { */ public void consume(Context ctx, Event event) throws Exception { - if (itemsToUpdate == null) { + if (Objects.isNull(itemsToUpdate)) { itemsToUpdate = new HashSet(); } @@ -76,7 +76,7 @@ public void consume(Context ctx, Event event) throws Exception { int et = event.getEventType(); - if (object != null && event.getObjectType() == Constants.ITEM) { + if (Objects.nonNull(object) && event.getObjectType() == Constants.ITEM) { //just update the object itemsToUpdate.add((Item)object); return; @@ -134,7 +134,7 @@ public void end(Context ctx) throws Exception { Set filtered = new HashSet(itemsToUpdate.size()); for (Item item : itemsToUpdate) { - if (item.getHandle() == null) { + if (Objects.isNull(item.getHandle())) { // probably submission item, skip continue; } @@ -156,7 +156,7 @@ public void end(Context ctx) throws Exception { itemsToUpdate = null; throw e; } finally { - if (anonymousContext != null) { + if (Objects.nonNull(anonymousContext)) { anonymousContext.complete(); } }