Fixes broken shim and tests for lifeycycle,cors,web,logging and acl c…

…ommands (#1768)

* Fixes broken shim and tests for lifeycycle,cors,web,logging and label commands.
Removes raw flag from acl command.

* Revert Changes of labels.py

* Undo requesterpays.py changes

gslib/commands/acl.py

@@ -357,7 +357,7 @@ def get_gcloud_storage_args(self):
         command_group = 'buckets'
       gcloud_storage_map = GcloudStorageMap(gcloud_command=[
           'storage', command_group, 'describe',
-          '--format=multi(acl:format=json)', '--raw'
+          '--format=multi(acl:format=json)'
       ],
                                             flag_map={})
 

gslib/commands/cors.py

@@ -147,9 +147,8 @@ class CorsCommand(Command):
               GcloudStorageMap(
                   gcloud_command=[
                       'storage', 'buckets', 'describe',
-                      '--format="gsutiljson[key=cors_config,empty=\' has no '
-                      'CORS configuration.\',empty_prefix_key=storage_url]"',
-                      '--raw'
+                      '--format=gsutiljson[key=cors_config,empty=\' has no '
+                      'CORS configuration.\',empty_prefix_key=storage_url]'
                   ],
                   flag_map={},
               ),

gslib/commands/lifecycle.py

@@ -164,9 +164,9 @@ def get_gcloud_storage_args(self):
                   GcloudStorageMap(
                       gcloud_command=[
                           'storage', 'buckets', 'describe',
-                          '--format="gsutiljson[key=lifecycle_config,empty=\' '
+                          '--format=gsutiljson[key=lifecycle_config,empty=\' '
                           'has no lifecycle configuration.\',empty_prefix_key='
-                          'storage_url]"', '--raw'
+                          'storage_url]'
                       ],
                       flag_map={},
                   ),

gslib/commands/logging.py

@@ -186,10 +186,10 @@ class LoggingCommand(Command):
           'get':
               GcloudStorageMap(
                   gcloud_command=[
-                      'storage', 'buckets', 'list',
-                      '--format="gsutiljson[key=logging_config,empty=\' has '
+                      'storage', 'buckets', 'describe',
+                      '--format=gsutiljson[key=logging_config,empty=\' has '
                       'no logging configuration.\',empty_prefix_key='
-                      'storage_url]"', '--raw'
+                      'storage_url]'
                   ],
                   flag_map={},
               ),

gslib/commands/web.py

@@ -162,9 +162,9 @@ class WebCommand(Command):
               GcloudStorageMap(
                   gcloud_command=[
                       'storage', 'buckets', 'describe',
-                      '--format="gsutiljson[key=website_config,empty=\' has '
+                      '--format=gsutiljson[key=website_config,empty=\' has '
                       'no website configuration.\',empty_prefix_key='
-                      'storage_url]"', '--raw'
+                      'storage_url]'
                   ],
                   flag_map={},
                   supports_output_translation=True,

gslib/tests/test_acl.py

@@ -85,7 +85,7 @@ def test_set_invalid_acl_object(self):
                             return_stderr=True,
                             expected_status=1)
     if self._use_gcloud_storage:
-      error_text = 'Found invalid JSON/YAML file'
+      error_text = 'Found invalid JSON/YAML'
     else:
       error_text = 'ArgumentException'
     self.assertIn(error_text, stderr)
@@ -98,7 +98,7 @@ def test_set_invalid_acl_bucket(self):
                             return_stderr=True,
                             expected_status=1)
     if self._use_gcloud_storage:
-      error_text = 'Found invalid JSON/YAML file'
+      error_text = 'Found invalid JSON/YAML'
     else:
       error_text = 'ArgumentException'
     self.assertIn(error_text, stderr)
@@ -112,7 +112,7 @@ def test_set_xml_acl_json_api_object(self):
                             expected_status=1)
 
     if self._use_gcloud_storage:
-      self.assertIn('Found invalid JSON/YAML file', stderr)
+      self.assertIn('Found invalid JSON/YAML', stderr)
       # XML not currently supported in gcloud storage.
     else:
       self.assertIn('ArgumentException', stderr)
@@ -126,7 +126,7 @@ def test_set_xml_acl_json_api_bucket(self):
                             return_stderr=True,
                             expected_status=1)
     if self._use_gcloud_storage:
-      self.assertIn('Found invalid JSON/YAML file', stderr)
+      self.assertIn('Found invalid JSON/YAML', stderr)
       # XML not currently supported in gcloud storage.
     else:
       self.assertIn('ArgumentException', stderr)
@@ -838,7 +838,7 @@ def test_shim_translates_acl_get_object(self):
         info_lines = '\n'.join(mock_log_handler.messages['info'])
         self.assertIn(('Gcloud Storage Command: {} storage objects describe'
                        ' --format=multi(acl:format=json)'
-                       ' --raw gs://bucket/object').format(
+                       ' gs://bucket/object').format(
                            shim_util._get_gcloud_binary_path('fake_dir')),
                       info_lines)
 
@@ -855,7 +855,7 @@ def test_shim_translates_acl_get_bucket(self):
         info_lines = '\n'.join(mock_log_handler.messages['info'])
         self.assertIn(('Gcloud Storage Command: {} storage buckets describe'
                        ' --format=multi(acl:format=json)'
-                       ' --raw gs://bucket').format(
+                       ' gs://bucket').format(
                            shim_util._get_gcloud_binary_path('fake_dir')),
                       info_lines)
 

gslib/tests/test_cors.py

@@ -87,8 +87,7 @@ class TestCors(testcase.GsUtilIntegrationTestCase):
 
   def setUp(self):
     super(TestCors, self).setUp()
-    self.no_cors = ('[]' if self._use_gcloud_storage else
-                    'has no CORS configuration')
+    self.no_cors = 'has no CORS configuration'
 
   def test_cors_translation(self):
     """Tests cors translation for various formats."""
@@ -148,7 +147,7 @@ def test_bad_cors(self):
                             return_stderr=True)
 
     if self._use_gcloud_storage:
-      self.assertIn('Found invalid JSON/YAML file', stderr)
+      self.assertIn('Found invalid JSON/YAML', stderr)
     else:
       self.assertNotIn('XML CORS data provided', stderr)
 

gslib/tests/test_label.py

@@ -284,10 +284,7 @@ def testChMinusDWorksWithoutExistingLabels(self):
     self.RunGsUtil(['label', 'ch', '-d', 'dummy-key', suri(bucket_uri)])
     stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)],
                             return_stdout=True)
-    if self._use_gcloud_storage:
-      self.assertIn('[]', stdout)
-    else:
-      self.assertIn('%s/ has no label configuration.' % suri(bucket_uri),
+    self.assertIn('%s/ has no label configuration.' % suri(bucket_uri),
                     stdout)
 
   def testTooFewArgumentsFails(self):

gslib/tests/test_lifecycle.py

@@ -76,7 +76,6 @@ class TestSetLifecycle(testcase.GsUtilIntegrationTestCase):
       '"age": 0, "isLive": false, "numNewerVersions": 0}}]}')
 
   no_lifecycle_config = 'has no lifecycle configuration.'
-  empty_lifecycle_config = '[]'
 
   def test_lifecycle_translation(self):
     """Tests lifecycle translation for various formats."""
@@ -96,21 +95,15 @@ def test_default_lifecycle(self):
     bucket_uri = self.CreateBucket()
     stdout = self.RunGsUtil(
         ['lifecycle', 'get', suri(bucket_uri)], return_stdout=True)
-    if self._use_gcloud_storage:
-      self.assertIn(self.empty_lifecycle_config, stdout)
-    else:
-      self.assertIn(self.no_lifecycle_config, stdout)
+    self.assertIn(self.no_lifecycle_config, stdout)
 
   def test_set_empty_lifecycle1(self):
     bucket_uri = self.CreateBucket()
     fpath = self.CreateTempFile(contents=self.empty_doc1.encode('ascii'))
     self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)])
     stdout = self.RunGsUtil(
         ['lifecycle', 'get', suri(bucket_uri)], return_stdout=True)
-    if self._use_gcloud_storage:
-      self.assertIn(self.empty_lifecycle_config, stdout)
-    else:
-      self.assertIn(self.no_lifecycle_config, stdout)
+    self.assertIn(self.no_lifecycle_config, stdout)
 
   def test_valid_lifecycle(self):
     bucket_uri = self.CreateBucket()
@@ -187,10 +180,7 @@ def test_set_lifecycle_and_reset(self):
     self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)])
     stdout = self.RunGsUtil(
         ['lifecycle', 'get', suri(bucket_uri)], return_stdout=True)
-    if self._use_gcloud_storage:
-      self.assertIn(self.empty_lifecycle_config, stdout)
-    else:
-      self.assertIn(self.no_lifecycle_config, stdout)
+    self.assertIn(self.no_lifecycle_config, stdout)
 
   def test_set_lifecycle_multi_buckets(self):
     """Tests setting lifecycle configuration on multiple buckets."""
@@ -290,9 +280,9 @@ def test_shim_translates_lifecycle_get_correctly(self):
         info_lines = '\n'.join(mock_log_handler.messages['info'])
         self.assertIn(
             ('Gcloud Storage Command: {} storage buckets'
-             ' describe --format="gsutiljson[key=lifecycle_config,empty=\' has'
-             ' no lifecycle configuration.\',empty_prefix_key=storage_url]"'
-             ' --raw {}').format(shim_util._get_gcloud_binary_path('fake_dir'),
+             ' describe --format=gsutiljson[key=lifecycle_config,empty=\' has'
+             ' no lifecycle configuration.\',empty_prefix_key=storage_url]'
+             ' {}').format(shim_util._get_gcloud_binary_path('fake_dir'),
                                  suri(bucket_uri)), info_lines)
 
   @mock.patch('gslib.commands.lifecycle.LifecycleCommand._SetLifecycleConfig',

gslib/tests/test_web.py

@@ -51,41 +51,29 @@ def test_full(self):
         ['-m', 'main', '-e', '404', suri(bucket_uri)])
     stdout = self.RunGsUtil(self._get_web_cmd + [suri(bucket_uri)],
                             return_stdout=True)
-    if self._use_gcloud_storage:
-      self.assertIn('"mainPageSuffix": "main"', stdout)
-      self.assertIn('"notFoundPage": "404"', stdout)
-    else:
-      self.assertEqual(json.loads(stdout), WEBCFG_FULL)
+    self.assertEqual(json.loads(stdout), WEBCFG_FULL)
 
   def test_main(self):
     bucket_uri = self.CreateBucket()
     self.RunGsUtil(self._set_web_cmd + ['-m', 'main', suri(bucket_uri)])
     stdout = self.RunGsUtil(self._get_web_cmd + [suri(bucket_uri)],
                             return_stdout=True)
-    if self._use_gcloud_storage:
-      self.assertEqual('{\n  "mainPageSuffix": "main"\n}\n', stdout)
-    else:
-      self.assertEqual(json.loads(stdout), WEBCFG_MAIN)
+
+    self.assertEqual(json.loads(stdout), WEBCFG_MAIN)
 
   def test_error(self):
     bucket_uri = self.CreateBucket()
     self.RunGsUtil(self._set_web_cmd + ['-e', '404', suri(bucket_uri)])
     stdout = self.RunGsUtil(self._get_web_cmd + [suri(bucket_uri)],
                             return_stdout=True)
-    if self._use_gcloud_storage:
-      self.assertEqual('{\n  "notFoundPage": "404"\n}\n', stdout)
-    else:
-      self.assertEqual(json.loads(stdout), WEBCFG_ERROR)
+    self.assertEqual(json.loads(stdout), WEBCFG_ERROR)
 
   def test_empty(self):
     bucket_uri = self.CreateBucket()
     self.RunGsUtil(self._set_web_cmd + [suri(bucket_uri)])
     stdout = self.RunGsUtil(self._get_web_cmd + [suri(bucket_uri)],
                             return_stdout=True)
-    if self._use_gcloud_storage:
-      self.assertEqual('[]\n', stdout)
-    else:
-      self.assertIn(WEBCFG_EMPTY, stdout)
+    self.assertIn(WEBCFG_EMPTY, stdout)
 
   def testTooFewArgumentsFails(self):
     """Ensures web commands fail with too few arguments."""
@@ -124,9 +112,9 @@ def test_shim_translates_get_command(self):
         info_lines = '\n'.join(mock_log_handler.messages['info'])
         self.assertIn(
             ('Gcloud Storage Command: {} storage buckets describe'
-             ' --format="gsutiljson[key=website_config,empty=\' has no website'
-             ' configuration.\',empty_prefix_key=storage_url]"'
-             ' --raw gs://bucket').format(
+             ' --format=gsutiljson[key=website_config,empty=\' has no website'
+             ' configuration.\',empty_prefix_key=storage_url]'
+             ' gs://bucket').format(
                  shim_util._get_gcloud_binary_path('fake_dir')), info_lines)
 
   @mock.patch.object(web.WebCommand, '_SetWeb', new=mock.Mock())