feat/arborist-sync

fence/__init__.py

@@ -40,7 +40,7 @@
 
 # Can't read config yet. Just set to debug for now, else no handlers.
 # Later, in app_config(), will actually set level based on config
-logger = get_logger(__name__, log_level='debug')
+logger = get_logger(__name__, log_level="debug")
 
 app = flask.Flask(__name__)
 CORS(app=app, headers=["content-type", "accept"], expose_headers="*")

fence/blueprints/data/blueprint.py

@@ -8,11 +8,7 @@
     IndexedFile,
     get_signed_url_for_file,
 )
-from fence.errors import (
-    Forbidden,
-    InternalError,
-    UserError,
-)
+from fence.errors import Forbidden, InternalError, UserError
 from fence.utils import is_valid_expiration
 from fence.rbac import check_arborist_auth
 

fence/jwt/token.py

@@ -169,9 +169,7 @@ def generate_signed_session_token(kid, private_key, expires_in, context=None):
         "jti": str(uuid.uuid4()),
         "context": context,
     }
-    logger.debug(
-        "issuing JWT session token\n" + json.dumps(claims, indent=4)
-    )
+    logger.debug("issuing JWT session token\n" + json.dumps(claims, indent=4))
     token = jwt.encode(claims, private_key, headers=headers, algorithm="RS256")
     token = to_unicode(token, "UTF-8")
 
@@ -271,12 +269,8 @@ def generate_signed_refresh_token(
     }
 
     if flask.current_app:
-        logger.info(
-            "issuing JWT refresh token with id [{}] to [{}]".format(jti, sub)
-        )
-        logger.debug(
-            "issuing JWT refresh token\n" + json.dumps(claims, indent=4)
-        )
+        logger.info("issuing JWT refresh token with id [{}] to [{}]".format(jti, sub))
+        logger.debug("issuing JWT refresh token\n" + json.dumps(claims, indent=4))
 
     token = jwt.encode(claims, private_key, headers=headers, algorithm="RS256")
     token = to_unicode(token, "UTF-8")
@@ -313,12 +307,8 @@ def generate_api_key(kid, private_key, user_id, expires_in, scopes, client_id):
         "jti": jti,
         "azp": client_id or "",
     }
-    logger.info(
-        "issuing JWT API key with id [{}] to [{}]".format(jti, sub)
-    )
-    logger.debug(
-        "issuing JWT API key\n" + json.dumps(claims, indent=4)
-    )
+    logger.info("issuing JWT API key with id [{}] to [{}]".format(jti, sub))
+    logger.debug("issuing JWT API key\n" + json.dumps(claims, indent=4))
     token = jwt.encode(claims, private_key, headers=headers, algorithm="RS256")
     logger.debug(str(token))
     token = to_unicode(token, "UTF-8")
@@ -391,12 +381,8 @@ def generate_signed_access_token(
         ] = linked_google_email
 
     if flask.current_app:
-        logger.info(
-            "issuing JWT access token with id [{}] to [{}]".format(jti, sub)
-        )
-        logger.debug(
-            "issuing JWT access token\n" + json.dumps(claims, indent=4)
-        )
+        logger.info("issuing JWT access token with id [{}] to [{}]".format(jti, sub))
+        logger.debug("issuing JWT access token\n" + json.dumps(claims, indent=4))
 
     token = jwt.encode(claims, private_key, headers=headers, algorithm="RS256")
     token = to_unicode(token, "UTF-8")
@@ -494,9 +480,7 @@ def generate_id_token(
     if nonce:
         claims["nonce"] = nonce
 
-    logger.info(
-        "issuing JWT ID token\n" + json.dumps(claims, indent=4)
-    )
+    logger.info("issuing JWT ID token\n" + json.dumps(claims, indent=4))
 
     token_options = {
         "iss": {"essential": True, "value": config.get("BASE_URL")},

fence/oidc/endpoints.py

@@ -59,9 +59,7 @@ def validate_authenticate_client(self):
             bcrypt.hashpw(client_secret.encode("utf-8"), hashed.encode("utf-8"))
             != hashed
         ):
-            logger.debug(
-                "client secret hash does not match stored secret hash"
-            )
+            logger.debug("client secret hash does not match stored secret hash")
             raise InvalidClientError(uri=self.uri)
 
         self._client = client

fence/rbac/client.py

@@ -258,9 +258,7 @@ def create_resource(self, parent_path, resource_json, overwrite=False):
                 msg = data["error"].get("message", msg)
             resource = resource_json.get("path", "/" + resource_json.get("name"))
             self.logger.error(
-                "could not create resource `{}` in arborist: {}".format(
-                    resource, msg
-                )
+                "could not create resource `{}` in arborist: {}".format(resource, msg)
             )
             raise ArboristError(data["error"])
         self.logger.info("created resource {}".format(resource_json["name"]))
@@ -444,15 +442,14 @@ def revoke_all_policies_for_user(self, username):
     def create_group(self, name, description="", users=None, policies=None):
         users = users or []
         policies = policies or []
-        data = {
-            "name": name,
-            "users": users,
-            "policies": policies,
-        }
+        data = {"name": name, "users": users, "policies": policies}
         if description:
             data["description"] = description
         response = requests.post(self._group_url, json=data)
         data = _request_get_json(response)
+        if response.status_code == 409:
+            # already exists; this is ok, but leave warning
+            self.logger.warn("group `{}` already exists in arborist".format(name))
         if response.status_code != 201:
             msg = data.get("error", "unhelpful response from arborist")
             self.logger.error("could not create group {}: {}".format(name, msg))

fence/resources/admin/admin_users.py

@@ -178,9 +178,7 @@ def delete_google_service_accounts_and_keys(current_session, gcm, gpg_email):
 
     def raise_unavailable(sae):
         raise UnavailableError(
-            "Error: Google unable to delete service account {}. Aborting".format(
-                sae
-            )
+            "Error: Google unable to delete service account {}. Aborting".format(sae)
         )
 
     for sae in service_account_emails:
@@ -203,11 +201,11 @@ def raise_unavailable(sae):
 
         logger.info(
             "Google service account with email {} successfully removed "
-            "from Google, along with all associated service account keys.".format(
-                sae
-            )
+            "from Google, along with all associated service account keys.".format(sae)
+        )
+        logger.debug(
+            "Attempting to clear service account records from Fence database..."
         )
-        logger.debug("Attempting to clear service account records from Fence database...")
         sa = (
             current_session.query(GoogleServiceAccount)
             .filter(GoogleServiceAccount.email == sae)
@@ -293,9 +291,8 @@ def raise_unavailable(gpg_email):
         )
         for row in gpg_to_gbag:
             current_session.delete(row)
-        logger.debug("Deleting rows in {}...".format(
-                UserGoogleAccountToProxyGroup.__tablename__
-            )
+        logger.debug(
+            "Deleting rows in {}...".format(UserGoogleAccountToProxyGroup.__tablename__)
         )
         uga_to_pg = (
             current_session.query(UserGoogleAccountToProxyGroup)
@@ -307,10 +304,7 @@ def raise_unavailable(gpg_email):
         )
         for row in uga_to_pg:
             current_session.delete(row)
-        logger.debug("Deleting rows in {}...".format(
-                UserGoogleAccount.__tablename__
-            )
-        )
+        logger.debug("Deleting rows in {}...".format(UserGoogleAccount.__tablename__))
         uga = (
             current_session.query(UserGoogleAccount)
             .filter(UserGoogleAccount.user_id == user.id)
@@ -323,9 +317,7 @@ def raise_unavailable(gpg_email):
         current_session.commit()
         logger.info(
             "Records for Google proxy group {} successfully cleared from Fence "
-            "database, along with associated user Google accounts.".format(
-                gpg_email
-            )
+            "database, along with associated user Google accounts.".format(gpg_email)
         )
         logger.info("Done with Google deletions.")
 
@@ -365,9 +357,7 @@ def delete_user(current_session, username):
 
         if google_proxy_group_from_fence_db:
             gpg_email = google_proxy_group_from_fence_db.email
-            logger.debug(
-                "Found Google proxy group in Fence db: {}".format(gpg_email)
-            )
+            logger.debug("Found Google proxy group in Fence db: {}".format(gpg_email))
         else:
             # Construct the proxy group name that would have been used
             # and check if it exists in cirrus, in case Fence db just
@@ -380,7 +370,9 @@ def delete_user(current_session, username):
             )
             google_proxy_group_from_google = gcm.get_group(pgname)
             gpg_email = (
-                google_proxy_group_from_google.get("email") if google_proxy_group_from_google else None
+                google_proxy_group_from_google.get("email")
+                if google_proxy_group_from_google
+                else None
             )
 
         if not gpg_email:

tests/data/test_data.py

@@ -636,7 +636,9 @@ def json(self):
         data_requests.post.return_value.status_code = 200
         arborist_requests.post.return_value = MockResponse({"auth": True})
         arborist_requests.post.return_value.status_code = 200
-        fence.blueprints.data.indexd.BlankIndex.init_multipart_upload.return_value = "test_uploadId"
+        fence.blueprints.data.indexd.BlankIndex.init_multipart_upload.return_value = (
+            "test_uploadId"
+        )
         headers = {
             "Authorization": "Bearer " + encoded_creds_jwt.jwt,
             "Content-Type": "application/json",
@@ -657,7 +659,9 @@ def json(self):
         assert "uploadId" in response.json
 
 
-def test_multipart_upload_presigned_url(app, client, auth_client, encoded_creds_jwt, user_client):
+def test_multipart_upload_presigned_url(
+    app, client, auth_client, encoded_creds_jwt, user_client
+):
     class MockResponse(object):
         def __init__(self, data, status_code=200):
             self.data = data
@@ -673,7 +677,9 @@ def json(self):
         "fence.rbac.client.requests", new_callable=mock.Mock
     )
 
-    fence.blueprints.data.indexd.BlankIndex.generate_aws_presigned_url_for_part = MagicMock()
+    fence.blueprints.data.indexd.BlankIndex.generate_aws_presigned_url_for_part = (
+        MagicMock()
+    )
     with data_requests_mocker as data_requests, arborist_requests_mocker as arborist_requests:
         data_requests.post.return_value = MockResponse(
             {
@@ -685,7 +691,9 @@ def json(self):
         data_requests.post.return_value.status_code = 200
         arborist_requests.post.return_value = MockResponse({"auth": True})
         arborist_requests.post.return_value.status_code = 200
-        fence.blueprints.data.indexd.BlankIndex.generate_aws_presigned_url_for_part.return_value = "test_presigned"
+        fence.blueprints.data.indexd.BlankIndex.generate_aws_presigned_url_for_part.return_value = (
+            "test_presigned"
+        )
         headers = {
             "Authorization": "Bearer " + encoded_creds_jwt.jwt,
             "Content-Type": "application/json",