BUG: mv() a file with recursive=True no longer working #394
Comments
|
Similar with |
|
Both Below is a reproduction of what I'm doing import s3fs
import os
common_key = os.path.join(bucket, data_path)
staging_key = os.path.join(common_key, "staging")
prod_key = os.path.join(common_key, "prod")
source = f"s3://{staging_key}"
destination = f"s3://{prod_key}"
s3 = s3fs.S3FileSystem()
s3.mv(source, destination, recursive=True)Produces: ---------------------------------------------------------------------------
NoSuchKey Traceback (most recent call last)
/usr/local/lib/python3.8/site-packages/s3fs/core.py in _call_s3(self, method, *akwarglist, **kwargs)
233 try:
--> 234 return await method(**additional_kwargs)
235 except S3_RETRYABLE_ERRORS as e:
/usr/local/lib/python3.8/site-packages/aiobotocore/client.py in _make_api_call(self, operation_name, api_params)
153 error_class = self.exceptions.from_code(error_code)
--> 154 raise error_class(parsed_response, operation_name)
155 else:
NoSuchKey: An error occurred (NoSuchKey) when calling the CopyObject operation: The specified key does not exist.
The above exception was the direct cause of the following exception:
FileNotFoundError Traceback (most recent call last)
<ipython-input-8-5a342e637684> in <module>
1 s3 = s3fs.S3FileSystem()
----> 2 s3.mv(source, destination, recursive=True)
/usr/local/lib/python3.8/site-packages/fsspec/spec.py in mv(self, path1, path2, recursive, maxdepth, **kwargs)
800 def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs):
801 """ Move file(s) from one location to another """
--> 802 self.copy(path1, path2, recursive=recursive, maxdepth=maxdepth)
803 self.rm(path1, recursive=recursive)
804
/usr/local/lib/python3.8/site-packages/fsspec/asyn.py in copy(self, path1, path2, recursive, **kwargs)
204 paths = self.expand_path(path1, recursive=recursive)
205 path2 = other_paths(paths, path2)
--> 206 maybe_sync(self._copy, self, paths, path2)
207
208 async def _pipe(self, path, value=None, **kwargs):
/usr/local/lib/python3.8/site-packages/fsspec/asyn.py in maybe_sync(func, self, *args, **kwargs)
98 if inspect.iscoroutinefunction(func):
99 # run the awaitable on the loop
--> 100 return sync(loop, func, *args, **kwargs)
101 else:
102 # just call the blocking function
/usr/local/lib/python3.8/site-packages/fsspec/asyn.py in sync(loop, func, callback_timeout, *args, **kwargs)
69 if error[0]:
70 typ, exc, tb = error[0]
---> 71 raise exc.with_traceback(tb)
72 else:
73 return result[0]
/usr/local/lib/python3.8/site-packages/fsspec/asyn.py in f()
53 if callback_timeout is not None:
54 future = asyncio.wait_for(future, callback_timeout)
---> 55 result[0] = await future
56 except Exception:
57 error[0] = sys.exc_info()
/usr/local/lib/python3.8/site-packages/fsspec/asyn.py in _copy(self, paths, path2, **kwargs)
197
198 async def _copy(self, paths, path2, **kwargs):
--> 199 await asyncio.gather(
200 *[self._cp_file(p1, p2, **kwargs) for p1, p2 in zip(paths, path2)]
201 )
/usr/local/lib/python3.8/site-packages/s3fs/core.py in _cp_file(self, path1, path2, **kwargs)
1350 if size <= gb5:
1351 # simple copy allowed for <5GB
-> 1352 await self._copy_basic(path1, path2, **kwargs)
1353 else:
1354 # serial multipart copy
/usr/local/lib/python3.8/site-packages/s3fs/core.py in _copy_basic(self, path1, path2, **kwargs)
1294 if ver1:
1295 copy_src["VersionId"] = ver1
-> 1296 await self._call_s3(
1297 self.s3.copy_object, kwargs, Bucket=buc2, Key=key2, CopySource=copy_src
1298 )
/usr/local/lib/python3.8/site-packages/s3fs/core.py in _call_s3(self, method, *akwarglist, **kwargs)
250 except Exception as e:
251 err = e
--> 252 raise translate_boto_error(err) from err
253
254 call_s3 = sync_wrapper(_call_s3)
FileNotFoundError: The specified key does not exist. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
When moving a file with
mv()while specifyingrecursive=True, this now fails with s3fs 0.5.1 or master, but was working with 0.4.This patch reproduces it in the tests:
The above is giving this test output on master:
Using a different test with the MinIO fixture setup as used in pyarrow (but pure s3fs test otherwise), I get a different error message ("ParamValidationError", see test in the error output below):
With released s3fs 0.5.1, this last test is actually working for the
mvstep, but fails to raise an error ons3.rm(source)(wheresourcedoes not exist aymore, as it was moved)The text was updated successfully, but these errors were encountered: