Skip to content

Commit 9457f3a

Browse files
dmitry-fadmitry-fa
authored
feat: add storage.upload(path) (#269)
* feat: add storage.upload(path) * feat: use mockito framework * feat: update upload to return blob * feat: do not parse response from writers for signed urls * fix reviewer's comments * Update Storage.upload() functionality after merge * feat: deprecate StorageRpc.write(), introduce StorageRpc.upload() * feat: deprecate StorageRpc.write(), introduce StorageRpc.upload() * rename upload to createFrom
1 parent 3e02b9c commit 9457f3a

File tree

11 files changed

+589
-52
lines changed

11 files changed

+589
-52
lines changed

‎google-cloud-storage/clirr-ignored-differences.xml

+5-10
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,19 @@
22
<!-- see http://mojo.codehaus.org/clirr-maven-plugin/examples/ignored-differences.html -->
33
<differences>
44
<difference>
5-
<className>com/google/cloud/storage/Storage</className>
6-
<method>com.google.cloud.storage.PostPolicyV4 generateSignedPostPolicyV4(com.google.cloud.storage.BlobInfo, long, java.util.concurrent.TimeUnit, com.google.cloud.storage.PostPolicyV4$PostFieldsV4, com.google.cloud.storage.PostPolicyV4$PostConditionsV4, com.google.cloud.storage.Storage$PostPolicyV4Option[])</method>
75
<differenceType>7012</differenceType>
8-
</difference>
9-
<difference>
106
<className>com/google/cloud/storage/Storage</className>
11-
<method>com.google.cloud.storage.PostPolicyV4 generateSignedPostPolicyV4(com.google.cloud.storage.BlobInfo, long, java.util.concurrent.TimeUnit, com.google.cloud.storage.PostPolicyV4$PostFieldsV4, com.google.cloud.storage.Storage$PostPolicyV4Option[])</method>
12-
<differenceType>7012</differenceType>
7+
<method>*.Blob createFrom(*.BlobInfo, java.nio.file.Path, *.Storage$BlobWriteOption[])</method>
138
</difference>
149
<difference>
15-
<className>com/google/cloud/storage/Storage</className>
16-
<method>com.google.cloud.storage.PostPolicyV4 generateSignedPostPolicyV4(com.google.cloud.storage.BlobInfo, long, java.util.concurrent.TimeUnit, com.google.cloud.storage.PostPolicyV4$PostConditionsV4, com.google.cloud.storage.Storage$PostPolicyV4Option[])</method>
1710
<differenceType>7012</differenceType>
11+
<className>com/google/cloud/storage/Storage</className>
12+
<method>*.Blob createFrom(*.BlobInfo, java.io.InputStream, *.Storage$BlobWriteOption[])</method>
1813
</difference>
1914
<difference>
20-
<className>com/google/cloud/storage/Storage</className>
21-
<method>com.google.cloud.storage.PostPolicyV4 generateSignedPostPolicyV4(com.google.cloud.storage.BlobInfo, long, java.util.concurrent.TimeUnit, com.google.cloud.storage.Storage$PostPolicyV4Option[])</method>
2215
<differenceType>7012</differenceType>
16+
<className>com/google/cloud/storage/spi/v1/StorageRpc</className>
17+
<method>*.StorageObject writeWithResponse(*.String, byte[], int, long, int, boolean)</method>
2318
</difference>
2419
<difference>
2520
<className>com/google/cloud/storage/BucketInfo$Builder</className>

‎google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java

+13-3
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import static com.google.cloud.RetryHelper.runWithRetries;
2020
import static java.util.concurrent.Executors.callable;
2121

22+
import com.google.api.services.storage.model.StorageObject;
2223
import com.google.cloud.BaseWriteChannel;
2324
import com.google.cloud.RestorableState;
2425
import com.google.cloud.RetryHelper;
@@ -47,6 +48,13 @@ class BlobWriteChannel extends BaseWriteChannel<StorageOptions, BlobInfo> {
4748
super(options, null, uploadId);
4849
}
4950

51+
// Contains metadata of the updated object or null if upload is not completed.
52+
private StorageObject storageObject;
53+
54+
StorageObject getStorageObject() {
55+
return storageObject;
56+
}
57+
5058
@Override
5159
protected void flushBuffer(final int length, final boolean last) {
5260
try {
@@ -55,9 +63,11 @@ protected void flushBuffer(final int length, final boolean last) {
5563
new Runnable() {
5664
@Override
5765
public void run() {
58-
getOptions()
59-
.getStorageRpcV1()
60-
.write(getUploadId(), getBuffer(), 0, getPosition(), length, last);
66+
storageObject =
67+
getOptions()
68+
.getStorageRpcV1()
69+
.writeWithResponse(
70+
getUploadId(), getBuffer(), 0, getPosition(), length, last);
6171
}
6272
}),
6373
getOptions().getRetrySettings(),

‎google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java

+118-4
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,11 @@
3939
import com.google.common.collect.Iterables;
4040
import com.google.common.collect.Lists;
4141
import com.google.common.io.BaseEncoding;
42+
import java.io.IOException;
4243
import java.io.InputStream;
4344
import java.io.Serializable;
4445
import java.net.URL;
46+
import java.nio.file.Path;
4547
import java.security.Key;
4648
import java.util.Arrays;
4749
import java.util.Collections;
@@ -1821,7 +1823,7 @@ public static Builder newBuilder() {
18211823
* Blob blob = storage.create(blobInfo);
18221824
* }</pre>
18231825
*
1824-
* @return a [@code Blob} with complete information
1826+
* @return a {@code Blob} with complete information
18251827
* @throws StorageException upon failure
18261828
*/
18271829
Blob create(BlobInfo blobInfo, BlobTargetOption... options);
@@ -1842,7 +1844,7 @@ public static Builder newBuilder() {
18421844
* Blob blob = storage.create(blobInfo, "Hello, World!".getBytes(UTF_8));
18431845
* }</pre>
18441846
*
1845-
* @return a [@code Blob} with complete information
1847+
* @return a {@code Blob} with complete information
18461848
* @throws StorageException upon failure
18471849
* @see <a href="https://cloud.google.com/storage/docs/hashes-etags">Hashes and ETags</a>
18481850
*/
@@ -1865,7 +1867,7 @@ public static Builder newBuilder() {
18651867
* Blob blob = storage.create(blobInfo, "Hello, World!".getBytes(UTF_8), 7, 5);
18661868
* }</pre>
18671869
*
1868-
* @return a [@code Blob} with complete information
1870+
* @return a {@code Blob} with complete information
18691871
* @throws StorageException upon failure
18701872
* @see <a href="https://cloud.google.com/storage/docs/hashes-etags">Hashes and ETags</a>
18711873
*/
@@ -1908,12 +1910,124 @@ Blob create(
19081910
* Blob blob = storage.create(blobInfo, content, BlobWriteOption.encryptionKey(encryptionKey));
19091911
* }</pre>
19101912
*
1911-
* @return a [@code Blob} with complete information
1913+
* @return a {@code Blob} with complete information
19121914
* @throws StorageException upon failure
19131915
*/
19141916
@Deprecated
19151917
Blob create(BlobInfo blobInfo, InputStream content, BlobWriteOption... options);
19161918

1919+
/**
1920+
* Uploads {@code path} to the blob using {@link #writer}. By default any MD5 and CRC32C values in
1921+
* the given {@code blobInfo} are ignored unless requested via the {@link
1922+
* BlobWriteOption#md5Match()} and {@link BlobWriteOption#crc32cMatch()} options. Folder upload is
1923+
* not supported.
1924+
*
1925+
* <p>Example of uploading a file:
1926+
*
1927+
* <pre>{@code
1928+
* String bucketName = "my-unique-bucket";
1929+
* String fileName = "readme.txt";
1930+
* BlobId blobId = BlobId.of(bucketName, fileName);
1931+
* BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build();
1932+
* storage.createFrom(blobInfo, Paths.get(fileName));
1933+
* }</pre>
1934+
*
1935+
* @param blobInfo blob to create
1936+
* @param path file to upload
1937+
* @param options blob write options
1938+
* @return a {@code Blob} with complete information
1939+
* @throws IOException on I/O error
1940+
* @throws StorageException on server side error
1941+
* @see #createFrom(BlobInfo, Path, int, BlobWriteOption...)
1942+
*/
1943+
Blob createFrom(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException;
1944+
1945+
/**
1946+
* Uploads {@code path} to the blob using {@link #writer} and {@code bufferSize}. By default any
1947+
* MD5 and CRC32C values in the given {@code blobInfo} are ignored unless requested via the {@link
1948+
* BlobWriteOption#md5Match()} and {@link BlobWriteOption#crc32cMatch()} options. Folder upload is
1949+
* not supported.
1950+
*
1951+
* <p>{@link #createFrom(BlobInfo, Path, BlobWriteOption...)} invokes this method with a buffer
1952+
* size of 15 MiB. Users can pass alternative values. Larger buffer sizes might improve the upload
1953+
* performance but require more memory. This can cause an OutOfMemoryError or add significant
1954+
* garbage collection overhead. Smaller buffer sizes reduce memory consumption, that is noticeable
1955+
* when uploading many objects in parallel. Buffer sizes less than 256 KiB are treated as 256 KiB.
1956+
*
1957+
* <p>Example of uploading a humongous file:
1958+
*
1959+
* <pre>{@code
1960+
* BlobId blobId = BlobId.of(bucketName, blobName);
1961+
* BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("video/webm").build();
1962+
*
1963+
* int largeBufferSize = 150 * 1024 * 1024;
1964+
* Path file = Paths.get("humongous.file");
1965+
* storage.createFrom(blobInfo, file, largeBufferSize);
1966+
* }</pre>
1967+
*
1968+
* @param blobInfo blob to create
1969+
* @param path file to upload
1970+
* @param bufferSize size of the buffer I/O operations
1971+
* @param options blob write options
1972+
* @return a {@code Blob} with complete information
1973+
* @throws IOException on I/O error
1974+
* @throws StorageException on server side error
1975+
*/
1976+
Blob createFrom(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options)
1977+
throws IOException;
1978+
1979+
/**
1980+
* Reads bytes from an input stream and uploads those bytes to the blob using {@link #writer}. By
1981+
* default any MD5 and CRC32C values in the given {@code blobInfo} are ignored unless requested
1982+
* via the {@link BlobWriteOption#md5Match()} and {@link BlobWriteOption#crc32cMatch()} options.
1983+
*
1984+
* <p>Example of uploading data with CRC32C checksum:
1985+
*
1986+
* <pre>{@code
1987+
* BlobId blobId = BlobId.of(bucketName, blobName);
1988+
* byte[] content = "Hello, world".getBytes(StandardCharsets.UTF_8);
1989+
* Hasher hasher = Hashing.crc32c().newHasher().putBytes(content);
1990+
* String crc32c = BaseEncoding.base64().encode(Ints.toByteArray(hasher.hash().asInt()));
1991+
* BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setCrc32c(crc32c).build();
1992+
* storage.createFrom(blobInfo, new ByteArrayInputStream(content), Storage.BlobWriteOption.crc32cMatch());
1993+
* }</pre>
1994+
*
1995+
* @param blobInfo blob to create
1996+
* @param content input stream to read from
1997+
* @param options blob write options
1998+
* @return a {@code Blob} with complete information
1999+
* @throws IOException on I/O error
2000+
* @throws StorageException on server side error
2001+
* @see #createFrom(BlobInfo, InputStream, int, BlobWriteOption...)
2002+
*/
2003+
Blob createFrom(BlobInfo blobInfo, InputStream content, BlobWriteOption... options)
2004+
throws IOException;
2005+
2006+
/**
2007+
* Reads bytes from an input stream and uploads those bytes to the blob using {@link #writer} and
2008+
* {@code bufferSize}. By default any MD5 and CRC32C values in the given {@code blobInfo} are
2009+
* ignored unless requested via the {@link BlobWriteOption#md5Match()} and {@link
2010+
* BlobWriteOption#crc32cMatch()} options.
2011+
*
2012+
* <p>{@link #createFrom(BlobInfo, InputStream, BlobWriteOption...)} )} invokes this method with a
2013+
* buffer size of 15 MiB. Users can pass alternative values. Larger buffer sizes might improve the
2014+
* upload performance but require more memory. This can cause an OutOfMemoryError or add
2015+
* significant garbage collection overhead. Smaller buffer sizes reduce memory consumption, that
2016+
* is noticeable when uploading many objects in parallel. Buffer sizes less than 256 KiB are
2017+
* treated as 256 KiB.
2018+
*
2019+
* @param blobInfo blob to create
2020+
* @param content input stream to read from
2021+
* @param bufferSize size of the buffer I/O operations
2022+
* @param options blob write options
2023+
* @return a {@code Blob} with complete information
2024+
* @throws IOException on I/O error
2025+
* @throws StorageException on server side error
2026+
*/
2027+
Blob createFrom(
2028+
BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options)
2029+
throws IOException;
2030+
19172031
/**
19182032
* Returns the requested bucket or {@code null} if not found.
19192033
*

‎google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java

+64
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import com.google.cloud.ReadChannel;
4949
import com.google.cloud.RetryHelper.RetryHelperException;
5050
import com.google.cloud.Tuple;
51+
import com.google.cloud.WriteChannel;
5152
import com.google.cloud.storage.Acl.Entity;
5253
import com.google.cloud.storage.HmacKey.HmacKeyMetadata;
5354
import com.google.cloud.storage.PostPolicyV4.ConditionV4Type;
@@ -70,12 +71,18 @@
7071
import com.google.common.io.BaseEncoding;
7172
import com.google.common.primitives.Ints;
7273
import java.io.ByteArrayInputStream;
74+
import java.io.IOException;
7375
import java.io.InputStream;
7476
import java.io.UnsupportedEncodingException;
7577
import java.net.MalformedURLException;
7678
import java.net.URI;
7779
import java.net.URL;
7880
import java.net.URLEncoder;
81+
import java.nio.ByteBuffer;
82+
import java.nio.channels.Channels;
83+
import java.nio.channels.ReadableByteChannel;
84+
import java.nio.file.Files;
85+
import java.nio.file.Path;
7986
import java.text.SimpleDateFormat;
8087
import java.util.Arrays;
8188
import java.util.Collections;
@@ -99,6 +106,9 @@ final class StorageImpl extends BaseService<StorageOptions> implements Storage {
99106

100107
private static final String STORAGE_XML_URI_HOST_NAME = "storage.googleapis.com";
101108

109+
private static final int DEFAULT_BUFFER_SIZE = 15 * 1024 * 1024;
110+
private static final int MIN_BUFFER_SIZE = 256 * 1024;
111+
102112
private static final Function<Tuple<Storage, Boolean>, Boolean> DELETE_FUNCTION =
103113
new Function<Tuple<Storage, Boolean>, Boolean>() {
104114
@Override
@@ -211,6 +221,60 @@ public StorageObject call() {
211221
}
212222
}
213223

224+
@Override
225+
public Blob createFrom(BlobInfo blobInfo, Path path, BlobWriteOption... options)
226+
throws IOException {
227+
return createFrom(blobInfo, path, DEFAULT_BUFFER_SIZE, options);
228+
}
229+
230+
@Override
231+
public Blob createFrom(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options)
232+
throws IOException {
233+
if (Files.isDirectory(path)) {
234+
throw new StorageException(0, path + " is a directory");
235+
}
236+
try (InputStream input = Files.newInputStream(path)) {
237+
return createFrom(blobInfo, input, bufferSize, options);
238+
}
239+
}
240+
241+
@Override
242+
public Blob createFrom(BlobInfo blobInfo, InputStream content, BlobWriteOption... options)
243+
throws IOException {
244+
return createFrom(blobInfo, content, DEFAULT_BUFFER_SIZE, options);
245+
}
246+
247+
@Override
248+
public Blob createFrom(
249+
BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options)
250+
throws IOException {
251+
252+
BlobWriteChannel blobWriteChannel;
253+
try (WriteChannel writer = writer(blobInfo, options)) {
254+
blobWriteChannel = (BlobWriteChannel) writer;
255+
uploadHelper(Channels.newChannel(content), writer, bufferSize);
256+
}
257+
StorageObject objectProto = blobWriteChannel.getStorageObject();
258+
return Blob.fromPb(this, objectProto);
259+
}
260+
261+
/*
262+
* Uploads the given content to the storage using specified write channel and the given buffer
263+
* size. This method does not close any channels.
264+
*/
265+
private static void uploadHelper(ReadableByteChannel reader, WriteChannel writer, int bufferSize)
266+
throws IOException {
267+
bufferSize = Math.max(bufferSize, MIN_BUFFER_SIZE);
268+
ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
269+
writer.setChunkSize(bufferSize);
270+
271+
while (reader.read(buffer) >= 0) {
272+
buffer.flip();
273+
writer.write(buffer);
274+
buffer.clear();
275+
}
276+
}
277+
214278
@Override
215279
public Bucket get(String bucket, BucketGetOption... options) {
216280
final com.google.api.services.storage.model.Bucket bucketPb = BucketInfo.of(bucket).toPb();

‎google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java

+24-1
Original file line numberDiff line numberDiff line change
@@ -725,11 +725,23 @@ public void write(
725725
long destOffset,
726726
int length,
727727
boolean last) {
728+
writeWithResponse(uploadId, toWrite, toWriteOffset, destOffset, length, last);
729+
}
730+
731+
@Override
732+
public StorageObject writeWithResponse(
733+
String uploadId,
734+
byte[] toWrite,
735+
int toWriteOffset,
736+
long destOffset,
737+
int length,
738+
boolean last) {
728739
Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_WRITE);
729740
Scope scope = tracer.withSpan(span);
741+
StorageObject updatedBlob = null;
730742
try {
731743
if (length == 0 && !last) {
732-
return;
744+
return updatedBlob;
733745
}
734746
GenericUrl url = new GenericUrl(uploadId);
735747
HttpRequest httpRequest =
@@ -750,6 +762,9 @@ public void write(
750762
range.append('*');
751763
}
752764
httpRequest.getHeaders().setContentRange(range.toString());
765+
if (last) {
766+
httpRequest.setParser(storage.getObjectParser());
767+
}
753768
int code;
754769
String message;
755770
IOException exception = null;
@@ -758,6 +773,13 @@ public void write(
758773
response = httpRequest.execute();
759774
code = response.getStatusCode();
760775
message = response.getStatusMessage();
776+
String contentType = response.getContentType();
777+
if (last
778+
&& (code == 200 || code == 201)
779+
&& contentType != null
780+
&& contentType.startsWith("application/json")) {
781+
updatedBlob = response.parseAs(StorageObject.class);
782+
}
761783
} catch (HttpResponseException ex) {
762784
exception = ex;
763785
code = ex.getStatusCode();
@@ -783,6 +805,7 @@ public void write(
783805
scope.close();
784806
span.end(HttpStorageRpcSpans.END_SPAN_OPTIONS);
785807
}
808+
return updatedBlob;
786809
}
787810

788811
@Override

0 commit comments

Comments
 (0)