diff --git a/vision/cloud-client/README.md b/vision/cloud-client/README.md index a62a7d7e44d..a182b9c4834 100644 --- a/vision/cloud-client/README.md +++ b/vision/cloud-client/README.md @@ -79,8 +79,17 @@ mvn exec:java -DDetect -Dexec.args="properties ./resources/city.jpg" mvn exec:java -DDetect -Dexec.args="web ./resources/landmark.jpg" ``` +#### Web Entities +``` +mvn exec:java -DDetect -Dexec.args="web-entities ./resources/landmark.jpg" +``` + +#### Web Entities Include Geo +``` +mvn exec:java -DDetect -Dexec.args="web-entities-include-geo ./resources/landmark.jpg" +``` + #### Crop ``` mvn exec:java -DDetect -Dexec.args="crop ./resources/landmark.jpg" ``` - diff --git a/vision/cloud-client/pom.xml b/vision/cloud-client/pom.xml index 1c18ee679be..8d392413ec2 100644 --- a/vision/cloud-client/pom.xml +++ b/vision/cloud-client/pom.xml @@ -40,7 +40,7 @@ com.google.cloud google-cloud-vision - 1.15.0 + 1.16.0 diff --git a/vision/cloud-client/resources/city.jpg b/vision/cloud-client/resources/city.jpg new file mode 100644 index 00000000000..b14282e7539 Binary files /dev/null and b/vision/cloud-client/resources/city.jpg differ diff --git a/vision/cloud-client/src/main/java/com/example/vision/Detect.java b/vision/cloud-client/src/main/java/com/example/vision/Detect.java index 69d41864941..2fdc007419a 100644 --- a/vision/cloud-client/src/main/java/com/example/vision/Detect.java +++ b/vision/cloud-client/src/main/java/com/example/vision/Detect.java @@ -14,7 +14,6 @@ * limitations under the License. */ - package com.example.vision; import com.google.cloud.vision.v1.AnnotateImageRequest; @@ -31,6 +30,7 @@ import com.google.cloud.vision.v1.Feature.Type; import com.google.cloud.vision.v1.Image; import com.google.cloud.vision.v1.ImageAnnotatorClient; +import com.google.cloud.vision.v1.ImageContext; import com.google.cloud.vision.v1.ImageSource; import com.google.cloud.vision.v1.LocationInfo; import com.google.cloud.vision.v1.Page; @@ -41,19 +41,24 @@ import com.google.cloud.vision.v1.WebDetection; import com.google.cloud.vision.v1.WebDetection.WebEntity; import com.google.cloud.vision.v1.WebDetection.WebImage; +import com.google.cloud.vision.v1.WebDetection.WebLabel; import com.google.cloud.vision.v1.WebDetection.WebPage; +import com.google.cloud.vision.v1.WebDetectionParams; import com.google.cloud.vision.v1.Word; + import com.google.protobuf.ByteString; + import java.io.FileInputStream; import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public class Detect { /** - * Detects entities,sentiment and syntax in a document using the Vision API. + * Detects entities, sentiment, and syntax in a document using the Vision API. * * @throws Exception on errors while closing the client. * @throws IOException on Input/Output errors. @@ -75,7 +80,7 @@ public static void argsHelper(String[] args, PrintStream out) throws Exception, "\tmvn exec:java -DDetect -Dexec.args=\" \"\n" + "Commands:\n" + "\tfaces | labels | landmarks | logos | text | safe-search | properties" - + "| web | crop \n" + + "| web | web-entities | web-entities-include-geo | crop \n" + "Path:\n\tA file path (ex: ./resources/wakeupcat.jpg) or a URI for a Cloud Storage " + "resource (gs://...)\n"); return; @@ -83,7 +88,6 @@ public static void argsHelper(String[] args, PrintStream out) throws Exception, String command = args[0]; String path = args.length > 1 ? args[1] : ""; - Detect app = new Detect(); if (command.equals("faces")) { if (path.startsWith("gs://")) { detectFacesGcs(path, out); @@ -134,6 +138,18 @@ public static void argsHelper(String[] args, PrintStream out) throws Exception, } else { detectWebDetections(path, out); } + } else if (command.equals("web-entities")) { + if (path.startsWith("gs://")) { + detectWebEntitiesGcs(path, out); + } else { + detectWebEntities(path, out); + } + } else if (command.equals("web-entities-include-geo")) { + if (path.startsWith("gs://")) { + detectWebEntitiesIncludeGeoResultsGcs(path, out); + } else { + detectWebEntitiesIncludeGeoResults(path, out); + } } else if (command.equals("crop")) { if (path.startsWith("gs://")) { detectCropHintsGcs(path, out); @@ -664,6 +680,7 @@ public static void detectPropertiesGcs(String gcsPath, PrintStream out) throws E } } + // [START vision_detect_safe_search] /** * Detects whether the specified image has features you would want to moderate. * @@ -697,17 +714,20 @@ public static void detectSafeSearch(String filePath, PrintStream out) throws Exc // For full list of available annotations, see http://g.co/cloud/vision/docs SafeSearchAnnotation annotation = res.getSafeSearchAnnotation(); out.printf( - "adult: %s\nmedical: %s\nspoofed: %s\nviolence: %s\n", + "adult: %s\nmedical: %s\nspoofed: %s\nviolence: %s\nracy: %s\n", annotation.getAdult(), annotation.getMedical(), annotation.getSpoof(), - annotation.getViolence()); + annotation.getViolence(), + annotation.getRacy()); } } } + // [END vision_detect_safe_search] + // [START vision_detect_safe_search_uri] /** - * Detects whether the specified remote image on Google Cloud Storage has features you would want + * Detects whether the specified image on Google Cloud Storage has features you would want * to moderate. * * @param gcsPath The path to the remote file on Google Cloud Storage to detect safe-search on. @@ -739,15 +759,18 @@ public static void detectSafeSearchGcs(String gcsPath, PrintStream out) throws E // For full list of available annotations, see http://g.co/cloud/vision/docs SafeSearchAnnotation annotation = res.getSafeSearchAnnotation(); out.printf( - "adult: %s\nmedical: %s\nspoofed: %s\nviolence: %s\n", + "adult: %s\nmedical: %s\nspoofed: %s\nviolence: %s\nracy: %s\n", annotation.getAdult(), annotation.getMedical(), annotation.getSpoof(), - annotation.getViolence()); + annotation.getViolence(), + annotation.getRacy()); } } } + // [END vision_detect_safe_search_uri] + // [START vision_detect_web] /** * Finds references to the specified image on the web. * @@ -788,6 +811,9 @@ public static void detectWebDetections(String filePath, PrintStream out) throws out.println(entity.getDescription() + " : " + entity.getEntityId() + " : " + entity.getScore()); } + for (WebLabel label : annotation.getBestGuessLabelsList()) { + out.format("\nBest guess label: %s", label.getLabel()); + } out.println("\nPages with matching images: Score\n=="); for (WebPage page : annotation.getPagesWithMatchingImagesList()) { out.println(page.getUrl() + " : " + page.getScore()); @@ -800,15 +826,21 @@ public static void detectWebDetections(String filePath, PrintStream out) throws for (WebImage image : annotation.getFullMatchingImagesList()) { out.println(image.getUrl() + " : " + image.getScore()); } + out.println("\nPages with visually similar images: Score\n=="); + for (WebImage image : annotation.getVisuallySimilarImagesList()) { + out.println(image.getUrl() + " : " + image.getScore()); + } } } } + // [END vision_detect_web] + // [START vision_detect_web_uri] /** - * Detects whether the specified remote image on Google Cloud Storage has features you would want - * to moderate. + * Detects whether the remote image on Google Cloud Storage has features you would want to + * moderate. * - * @param gcsPath The path to the remote file on Google Cloud Storage to detect safe-search on. + * @param gcsPath The path to the remote on Google Cloud Storage file to detect web annotations. * @param out A {@link PrintStream} to write the results to. * @throws Exception on errors while closing the client. * @throws IOException on Input/Output errors. @@ -844,6 +876,9 @@ public static void detectWebDetectionsGcs(String gcsPath, PrintStream out) throw out.println(entity.getDescription() + " : " + entity.getEntityId() + " : " + entity.getScore()); } + for (WebLabel label : annotation.getBestGuessLabelsList()) { + out.format("\nBest guess label: %s", label.getLabel()); + } out.println("\nPages with matching images: Score\n=="); for (WebPage page : annotation.getPagesWithMatchingImagesList()) { out.println(page.getUrl() + " : " + page.getScore()); @@ -856,9 +891,187 @@ public static void detectWebDetectionsGcs(String gcsPath, PrintStream out) throw for (WebImage image : annotation.getFullMatchingImagesList()) { out.println(image.getUrl() + " : " + image.getScore()); } + out.println("\nPages with visually similar images: Score\n=="); + for (WebImage image : annotation.getVisuallySimilarImagesList()) { + out.println(image.getUrl() + " : " + image.getScore()); + } } } } + // [END vision_detect_web_uri] + + /** + * Find web entities given a local image. + * @param filePath The path of the image to detect. + * @param out A {@link PrintStream} to write the results to. + * @throws Exception on errors while closing the client. + * @throws IOException on Input/Output errors. + */ + public static void detectWebEntities(String filePath, PrintStream out) throws Exception, + IOException { + // Instantiates a client + try (ImageAnnotatorClient client = ImageAnnotatorClient.create()) { + // Read in the local image + ByteString contents = ByteString.readFrom(new FileInputStream(filePath)); + + // Build the image + Image image = Image.newBuilder().setContent(contents).build(); + + // Create the request with the image and the specified feature: web detection + AnnotateImageRequest request = AnnotateImageRequest.newBuilder() + .addFeatures(Feature.newBuilder().setType(Type.WEB_DETECTION)) + .setImage(image) + .build(); + + // Perform the request + BatchAnnotateImagesResponse response = client.batchAnnotateImages(Arrays.asList(request)); + + // Display the results + response.getResponsesList().stream() + .forEach(r -> r.getWebDetection().getWebEntitiesList().stream() + .forEach(entity -> { + out.format("Description: %s\n", entity.getDescription()); + out.format("Score: %f\n", entity.getScore()); + })); + } + } + + /** + * Find web entities given the remote image on Google Cloud Storage. + * @param gcsPath The path to the remote file on Google Cloud Storage to detect web entities. + * @param out A {@link PrintStream} to write the results to. + * @throws Exception on errors while closing the client. + * @throws IOException on Input/Output errors. + */ + public static void detectWebEntitiesGcs(String gcsPath, PrintStream out) throws Exception, + IOException { + // Instantiates a client + try (ImageAnnotatorClient client = ImageAnnotatorClient.create()) { + // Set the image source to the given gs uri + ImageSource imageSource = ImageSource.newBuilder() + .setGcsImageUri(gcsPath) + .build(); + // Build the image + Image image = Image.newBuilder().setSource(imageSource).build(); + + // Create the request with the image and the specified feature: web detection + AnnotateImageRequest request = AnnotateImageRequest.newBuilder() + .addFeatures(Feature.newBuilder().setType(Type.WEB_DETECTION)) + .setImage(image) + .build(); + + // Perform the request + BatchAnnotateImagesResponse response = client.batchAnnotateImages(Arrays.asList(request)); + + // Display the results + response.getResponsesList().stream() + .forEach(r -> r.getWebDetection().getWebEntitiesList().stream() + .forEach(entity -> { + System.out.format("Description: %s\n", entity.getDescription()); + System.out.format("Score: %f\n", entity.getScore()); + })); + } + } + + // [START vision_web_entities_include_geo_results] + /** + * Find web entities given a local image. + * @param filePath The path of the image to detect. + * @param out A {@link PrintStream} to write the results to. + * @throws Exception on errors while closing the client. + * @throws IOException on Input/Output errors. + */ + public static void detectWebEntitiesIncludeGeoResults(String filePath, PrintStream out) throws + Exception, IOException { + // Instantiates a client + try (ImageAnnotatorClient client = ImageAnnotatorClient.create()) { + // Read in the local image + ByteString contents = ByteString.readFrom(new FileInputStream(filePath)); + + // Build the image + Image image = Image.newBuilder().setContent(contents).build(); + + // Enable `IncludeGeoResults` + WebDetectionParams webDetectionParams = WebDetectionParams.newBuilder() + .setIncludeGeoResults(true) + .build(); + + // Set the parameters for the image + ImageContext imageContext = ImageContext.newBuilder() + .setWebDetectionParams(webDetectionParams) + .build(); + + // Create the request with the image, imageContext, and the specified feature: web detection + AnnotateImageRequest request = AnnotateImageRequest.newBuilder() + .addFeatures(Feature.newBuilder().setType(Type.WEB_DETECTION)) + .setImage(image) + .setImageContext(imageContext) + .build(); + + // Perform the request + BatchAnnotateImagesResponse response = client.batchAnnotateImages(Arrays.asList(request)); + + // Display the results + response.getResponsesList().stream() + .forEach(r -> r.getWebDetection().getWebEntitiesList().stream() + .forEach(entity -> { + out.format("Description: %s\n", entity.getDescription()); + out.format("Score: %f\n", entity.getScore()); + })); + } + } + // [END vision_web_entities_include_geo_results] + + // [START vision_web_entities_include_geo_results_uri] + /** + * Find web entities given the remote image on Google Cloud Storage. + * @param gcsPath The path to the remote file on Google Cloud Storage to detect web entities with + * geo results. + * @param out A {@link PrintStream} to write the results to. + * @throws Exception on errors while closing the client. + * @throws IOException on Input/Output errors. + */ + public static void detectWebEntitiesIncludeGeoResultsGcs(String gcsPath, PrintStream out) throws + Exception, IOException { + // Instantiates a client + try (ImageAnnotatorClient client = ImageAnnotatorClient.create()) { + // Set the image source to the given gs uri + ImageSource imageSource = ImageSource.newBuilder() + .setGcsImageUri(gcsPath) + .build(); + // Build the image + Image image = Image.newBuilder().setSource(imageSource).build(); + + // Enable `IncludeGeoResults` + WebDetectionParams webDetectionParams = WebDetectionParams.newBuilder() + .setIncludeGeoResults(true) + .build(); + + // Set the parameters for the image + ImageContext imageContext = ImageContext.newBuilder() + .setWebDetectionParams(webDetectionParams) + .build(); + + // Create the request with the image, imageContext, and the specified feature: web detection + AnnotateImageRequest request = AnnotateImageRequest.newBuilder() + .addFeatures(Feature.newBuilder().setType(Type.WEB_DETECTION)) + .setImage(image) + .setImageContext(imageContext) + .build(); + + // Perform the request + BatchAnnotateImagesResponse response = client.batchAnnotateImages(Arrays.asList(request)); + + // Display the results + response.getResponsesList().stream() + .forEach(r -> r.getWebDetection().getWebEntitiesList().stream() + .forEach(entity -> { + out.format("Description: %s\n", entity.getDescription()); + out.format("Score: %f\n", entity.getScore()); + })); + } + } + // [END vision_web_entities_include_geo_results_uri] /** * Suggests a region to crop to for a local file. @@ -937,6 +1150,7 @@ public static void detectCropHintsGcs(String gcsPath, PrintStream out) throws Ex } } + // [START vision_detect_document] /** * Performs document text detection on a local image file. * @@ -980,24 +1194,30 @@ public static void detectDocumentText(String filePath, PrintStream out) throws E String wordText = ""; for (Symbol symbol: word.getSymbolsList()) { wordText = wordText + symbol.getText(); + out.format("Symbol text: %s (confidence: %f)\n", symbol.getText(), + symbol.getConfidence()); } - paraText = paraText + wordText; + out.format("Word text: %s (confidence: %f)\n\n", wordText, word.getConfidence()); + paraText = String.format("%s %s", paraText, wordText); } // Output Example using Paragraph: - out.println("Paragraph: \n" + paraText); - out.println("Bounds: \n" + para.getBoundingBox() + "\n"); + out.println("\nParagraph: \n" + paraText); + out.format("Paragraph Confidence: %f\n", para.getConfidence()); blockText = blockText + paraText; } pageText = pageText + blockText; } } + out.println("\nComplete annotation:"); out.println(annotation.getText()); } } } + // [END vision_detect_document] + // [START vision_detect_document_uri] /** - * Performs document text detection on a local image file on Google Cloud Storage. + * Performs document text detection on a remote image on Google Cloud Storage. * * @param gcsPath The path to the remote file on Google Cloud Storage to detect document text on. * @param out A {@link PrintStream} to write the results to. @@ -1037,19 +1257,24 @@ public static void detectDocumentTextGcs(String gcsPath, PrintStream out) throws String wordText = ""; for (Symbol symbol: word.getSymbolsList()) { wordText = wordText + symbol.getText(); + out.format("Symbol text: %s (confidence: %f)\n", symbol.getText(), + symbol.getConfidence()); } - paraText = paraText + wordText; + out.format("Word text: %s (confidence: %f)\n\n", wordText, word.getConfidence()); + paraText = String.format("%s %s", paraText, wordText); } // Output Example using Paragraph: - out.println("Paragraph: \n" + paraText); - out.println("Bounds: \n" + para.getBoundingBox() + "\n"); + out.println("\nParagraph: \n" + paraText); + out.format("Paragraph Confidence: %f\n", para.getConfidence()); blockText = blockText + paraText; } pageText = pageText + blockText; } } + out.println("\nComplete annotation:"); out.println(annotation.getText()); } } } + // [END vision_detect_document_uri] } diff --git a/vision/cloud-client/src/test/java/com/example/vision/DetectIT.java b/vision/cloud-client/src/test/java/com/example/vision/DetectIT.java index 5efdcf3ecff..d85d0d2d935 100644 --- a/vision/cloud-client/src/test/java/com/example/vision/DetectIT.java +++ b/vision/cloud-client/src/test/java/com/example/vision/DetectIT.java @@ -21,6 +21,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; + import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -33,7 +34,6 @@ public class DetectIT { private ByteArrayOutputStream bout; private PrintStream out; - private Detect app; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); private static final String BUCKET = PROJECT_ID; @@ -42,7 +42,6 @@ public void setUp() throws IOException { bout = new ByteArrayOutputStream(); out = new PrintStream(bout); System.setOut(out); - app = new Detect(); } @After @@ -186,6 +185,7 @@ public void testSafeSearch() throws Exception { // Assert String got = bout.toString(); assertThat(got).contains("adult: VERY_UNLIKELY"); + assertThat(got).contains("racy: UNLIKELY"); } @Test @@ -197,6 +197,7 @@ public void testSafeSearchGcs() throws Exception { // Assert String got = bout.toString(); assertThat(got).contains("adult: VERY_UNLIKELY"); + assertThat(got).contains("racy: UNLIKELY"); } @Test @@ -235,7 +236,8 @@ public void detectWebAnnotations() throws Exception { // Assert String got = bout.toString(); - assertThat(got).contains("Palace"); + assertThat(got).contains("Palace of Fine Arts Theatre"); + assertThat(got).contains("Best guess label: palace of fine arts"); } @Test @@ -246,7 +248,51 @@ public void detectWebAnnotationsGcs() throws Exception { // Assert String got = bout.toString(); - assertThat(got).contains("Palace"); + assertThat(got).contains("Palace of Fine Arts Theatre"); + assertThat(got).contains("Best guess label: palace of fine arts"); + } + + @Test + public void testDetectWebEntities() throws Exception { + // Act + String[] args = {"web-entities", "./resources/city.jpg"}; + Detect.argsHelper(args, out); + + // Assert + String got = bout.toString(); + assertThat(got).doesNotContain("Zepra"); + } + + @Test + public void testDetectWebEntitiesGcs() throws Exception { + // Act + String[] args = {"web-entities", "gs://" + BUCKET + "/vision/landmark.jpg"}; + Detect.argsHelper(args, out); + + String got = bout.toString(); + assertThat(got).contains("Description: Palace of Fine Arts Theatre"); + } + + @Test + public void testDetectWebEntitiesIncludeGeoResults() throws Exception { + // Act + String[] args = {"web-entities-include-geo", "./resources/city.jpg"}; + Detect.argsHelper(args, out); + + // Assert + String got = bout.toString(); + // Note: entities and labels can change over time. + assertThat(got).doesNotContain("Error"); + } + + @Test + public void testDetectWebEntitiesIncludeGeoResultsGcs() throws Exception { + // Act + String[] args = {"web-entities-include-geo", "gs://" + BUCKET + "/vision/landmark.jpg"}; + Detect.argsHelper(args, out); + + String got = bout.toString(); + assertThat(got).contains("Description: Palace of Fine Arts Theatre"); } @Test @@ -285,6 +331,7 @@ public void testDocumentText() throws Exception { String got = bout.toString(); assertThat(got).contains("After preparation is complete, the "); assertThat(got).contains("37%"); + assertThat(got).contains("Word text: class (confidence:"); } @Test @@ -297,5 +344,6 @@ public void testDocumentTextGcs() throws Exception { String got = bout.toString(); assertThat(got).contains("After preparation is complete, the "); assertThat(got).contains("37%"); + assertThat(got).contains("Word text: class (confidence:"); } }