Skip to content

Commit

Permalink
Merge pull request #993 from WildMeOrg/872_matchingset_opensearch
Browse files Browse the repository at this point in the history 
matchingSet built using OpenSearch
  • Loading branch information
@holmbergius
holmbergius authored Feb 18, 2025
2 parents 065b67a + 2d740db commit d54b991
Show file tree
Hide file tree
Showing 8 changed files with 338 additions and 319 deletions.
491 changes: 250 additions & 241 deletions src/main/java/org/ecocean/Annotation.java
View file

Large diffs are not rendered by default.

35 changes: 25 additions & 10 deletions src/main/java/org/ecocean/Encounter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4363,20 +4363,35 @@ public void run() {
bgShepherd.beginDBTransaction();
try {
Encounter enc = bgShepherd.getEncounter(encId);
if ((enc == null) || !enc.hasMarkedIndividual()) {
if ((enc == null) || (!enc.hasMarkedIndividual() && !enc.hasAnnotations())) {
// bgShepherd.rollbackAndClose();
executor.shutdown();
return;
}
MarkedIndividual indiv = enc.getIndividual();
System.out.println("opensearchIndexDeep() background indexing indiv " +
indiv.getId() + " via enc " + encId);
try {
indiv.opensearchIndex();
} catch (Exception ex) {
System.out.println("opensearchIndexDeep() background indexing " +
indiv.getId() + " FAILED: " + ex.toString());
ex.printStackTrace();
if (indiv != null) {
System.out.println("opensearchIndexDeep() background indexing indiv " +
indiv.getId() + " via enc " + encId);
try {
indiv.opensearchIndex();
} catch (Exception ex) {
System.out.println("opensearchIndexDeep() background indexing " +
indiv.getId() + " FAILED: " + ex.toString());
ex.printStackTrace();
}
}
if (enc.hasAnnotations()) {
for (Annotation ann : enc.getAnnotations()) {
System.out.println("opensearchIndexDeep() background indexing annot " +
ann.getId() + " via enc " + encId);
try {
ann.opensearchIndex();
} catch (Exception ex) {
System.out.println("opensearchIndexDeep() background indexing " +
ann.getId() + " FAILED: " + ex.toString());
ex.printStackTrace();
}
}
}
} catch (Exception e) {
System.out.println("opensearchIndexDeep() backgrounding Encounter " + encId +
Expand All @@ -4390,7 +4405,7 @@ public void run() {
executor.shutdown();
}
};
System.out.println("opensearchIndexDeep() begin backgrounding indiv for " + this);
System.out.println("opensearchIndexDeep() begin backgrounding for " + this);
executor.execute(rn);
System.out.println("opensearchIndexDeep() [foreground] finished for Encounter " + encId);
}
Expand Down
19 changes: 19 additions & 0 deletions src/main/java/org/ecocean/OpenSearch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,25 @@ public JSONObject queryPit(String indexName, final JSONObject query, int numFrom
return new JSONObject(rtn);
}

// just return the actual hit results
// note: each object in the array has _id but actual doc is in _source!!
public static JSONArray getHits(JSONObject queryResults) {
JSONArray failed = new JSONArray();

if (queryResults == null) return failed;
JSONObject outerHits = queryResults.optJSONObject("hits");
if (outerHits == null) {
System.out.println("could not find (outer) hits");
return failed;
}
JSONArray hits = outerHits.optJSONArray("hits");
if (hits == null) {
System.out.println("could not find hits");
return failed;
}
return hits;
}

// https://opensearch.org/docs/2.3/opensearch/search/paginate/
public JSONObject queryRawScroll(String indexName, final JSONObject query, int pageSize)
throws IOException {
Expand Down
20 changes: 20 additions & 0 deletions src/main/java/org/ecocean/Shepherd.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2063,6 +2063,26 @@ public Iterator getAllAnnotationsNoQuery() {
}
}

// note: where clause can also contain " ORDER BY xxx"
public Iterator getAnnotationsFilter(String jdoWhereClause) {
Query query = null;

try {
query = pm.newQuery("SELECT FROM org.ecocean.Annotation WHERE " + jdoWhereClause);
Collection c = (Collection)(query.execute());
List list = new ArrayList(c);
Iterator it = list.iterator();
query.closeAll();
return it;
} catch (Exception npe) {
System.out.println(
"Error encountered when trying to execute getAllAnnotationsFilter. Returning a null iterator.");
npe.printStackTrace();
if (query != null) query.closeAll();
return null;
}
}

public Iterator<Encounter> getAllAnnotations(String order) {
Extent extClass = pm.getExtent(Annotation.class, true);
Query q = pm.newQuery(extClass);
Expand Down
4 changes: 4 additions & 0 deletions src/main/java/org/ecocean/api/SearchApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response)
} else if ((searchQueryId == null) && !OpenSearch.isValidIndexName(indexName)) {
response.setStatus(404);
res.put("error", "unknown index");
} else if ("annotation".equals(indexName) && !currentUser.isAdmin(myShepherd)) {
// per discussion with jh today, api exposure of annotations admin-only currently
response.setStatus(403);
res.put("error", 403);
} else if ((query == null) && !"POST".equals(request.getMethod())) {
response.setStatus(405);
res.put("error", "method not allowed");
Expand Down
83 changes: 18 additions & 65 deletions src/main/java/org/ecocean/ia/plugin/WildbookIAM.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
import javax.servlet.ServletContextEvent;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.ecocean.acm.AcmUtil;
import org.ecocean.Annotation;
import org.ecocean.cache.CachedQuery;
import org.ecocean.cache.QueryCache;
import org.ecocean.cache.QueryCacheFactory;
import org.ecocean.Annotation;
import org.ecocean.ia.IA;
import org.ecocean.ia.Task;
import org.ecocean.media.*;
Expand Down Expand Up @@ -82,48 +82,6 @@ public void prime() {
IBEISIA.setIAPrimed(true);
}

public void _OLD_prime() {
IA.log("INFO: WildbookIAM.prime(" + this.context + ") called");
IBEISIA.setIAPrimed(false);
if (!isEnabled()) return;
final List<String> iaAnnotIds = iaAnnotationIds();
final List<String> iaImageIds = iaImageIds();
Runnable r = new Runnable() {
public void run() {
Shepherd myShepherd = new Shepherd(context);
myShepherd.setAction("WildbookIAM.prime");
myShepherd.beginDBTransaction();
ArrayList<Annotation> matchingSet = Annotation.getAllMatchAgainstTrue(myShepherd);
ArrayList<Annotation> sendAnns = new ArrayList<Annotation>();
ArrayList<MediaAsset> mas = new ArrayList<MediaAsset>();
for (Annotation ann : matchingSet) {
if (iaAnnotIds.contains(ann.getAcmId())) continue; // no need to send
sendAnns.add(ann);
MediaAsset ma = ann.getDerivedMediaAsset();
if (ma == null) ma = ann.getMediaAsset();
if (ma == null) continue;
if (iaImageIds.contains(ma.getAcmId())) continue;
mas.add(ma);
}
IA.log("INFO: WildbookIAM.prime(" + context + ") sending " + sendAnns.size() +
" annots (of " + matchingSet.size() + ") and " + mas.size() + " images");
try {
// think we can checkFirst on both of these -- no need to re-send anything during priming
sendMediaAssets(mas, true);
sendAnnotations(sendAnns, true, myShepherd);
} catch (Exception ex) {
IA.log("ERROR: WildbookIAM.prime() failed due to " + ex.toString());
ex.printStackTrace();
}
myShepherd.commitDBTransaction(); // MAs and annots may have had acmIds changed
myShepherd.closeDBTransaction();
IBEISIA.setIAPrimed(true);
IA.log("INFO: WildbookIAM.prime(" + context + ") complete");
}
};
new Thread(r).start();
}

/*
note: sendMediaAssets() and sendAnnotations() need to be *batched* now in small chunks, particularly sendMediaAssets().
this is because we **must** get the return value from the POST, in order that we can map the corresponding (returned) acmId values. if we
Expand Down Expand Up @@ -317,33 +275,28 @@ public List<String> iaAnnotationIds() {
public static List<String> iaAnnotationIds(String context) {
List<String> ids = new ArrayList<String>();
JSONArray jids = null;
String cacheName="iaAnnotationIds";
String cacheName = "iaAnnotationIds";

try {


QueryCache qc = QueryCacheFactory.getQueryCache(context);
QueryCache qc = QueryCacheFactory.getQueryCache(context);
if (qc.getQueryByName(cacheName) != null &&
System.currentTimeMillis() <
qc.getQueryByName(cacheName).getNextExpirationTimeout()) {

org.datanucleus.api.rest.orgjson.JSONObject jobj = Util.toggleJSONObject(qc.getQueryByName(cacheName).getJSONSerializedQueryResult());
jids=Util.toggleJSONArray(jobj.getJSONArray("iaAnnotationIds"));
}
else {
jids = apiGetJSONArray("/api/annot/json/", context);
if(jids!=null) {
org.datanucleus.api.rest.orgjson.JSONObject jobj =new org.datanucleus.api.rest.orgjson.JSONObject();
jobj.put("iaAnnotationIds",Util.toggleJSONArray(jids));
CachedQuery cq = new CachedQuery(cacheName, Util.toggleJSONObject(jobj));
cq.nextExpirationTimeout = System.currentTimeMillis() + (15*60*1000);
qc.addCachedQuery(cq);
}

org.datanucleus.api.rest.orgjson.JSONObject jobj = Util.toggleJSONObject(
qc.getQueryByName(cacheName).getJSONSerializedQueryResult());
jids = Util.toggleJSONArray(jobj.getJSONArray("iaAnnotationIds"));
} else {
jids = apiGetJSONArray("/api/annot/json/", context);
if (jids != null) {
org.datanucleus.api.rest.orgjson.JSONObject jobj =
new org.datanucleus.api.rest.orgjson.JSONObject();
jobj.put("iaAnnotationIds", Util.toggleJSONArray(jids));
CachedQuery cq = new CachedQuery(cacheName, Util.toggleJSONObject(jobj));
cq.nextExpirationTimeout = System.currentTimeMillis() + (15 * 60 * 1000);
qc.addCachedQuery(cq);
}
}


}
catch (Exception ex) {
} catch (Exception ex) {
ex.printStackTrace();
IA.log("ERROR: WildbookIAM.iaAnnotationIds() returning empty; failed due to " +
ex.toString());
Expand Down
3 changes: 1 addition & 2 deletions src/main/java/org/ecocean/identity/IAQueryCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public static int buildTargetAnnotationsCache(String context, ArrayList<Annotati
// qanns can "in theory" contain more than one annot, but we arent ready for that universe yet anyway...
if ((qanns == null) || (qanns.size() < 1)) return -1; // :(
// we want *non* excluding version of this:
ArrayList<Annotation> anns = qanns.get(0).getMatchingSetForTaxonomy(myShepherd, null);
ArrayList<Annotation> anns = qanns.get(0).getMatchingSet(myShepherd);
if (anns == null) return -2;
JSONObject jdata = new JSONObject();
JSONArray idArr = new JSONArray();
Expand Down Expand Up @@ -149,7 +149,6 @@ public static JSONObject addTargetAnnotation(String context, Annotation ann,
log("WARNING: addTargetAnnotation() found empty current arrays! weird.");
return null;
}

String indivId = ann.findIndividualId(myShepherd);
// cheap fix to handle name-conflict potential: we dont add an annot which is already on list in cache!
if (indivId == null) {
Expand Down
2 changes: 1 addition & 1 deletion src/main/webapp/appadmin/opensearchSync.jsp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ if (endNum > 0) {
if (indexName.equals("encounter")) {
itr = myShepherd.getAllEncounters("catalogNumber");
} else if (indexName.equals("annotation")) {
itr = myShepherd.getAllAnnotations("id");
itr = myShepherd.getAnnotationsFilter("matchAgainst == true && acmId != null ORDER BY id");
} else if (indexName.equals("individual")) {
itr = myShepherd.getAllMarkedIndividuals();
}
Expand Down

0 comments on commit d54b991

Please sign in to comment.