Merge pull request #250 from ImMin5/master

Add cost_data_keys filter at data_source 'get' and 'list' api

Author: ImMin5

src/spaceone/cost_analysis/interface/grpc/data_source.py

@@ -106,27 +106,15 @@ def sync(self, request, context):
 
     def get(self, request, context):
         params, metadata = self.parse_request(request, context)
-
-        with self.locator.get_service(
-            "DataSourceService", metadata
-        ) as data_source_service:
-            return self.locator.get_info(
-                "DataSourceInfo", data_source_service.get(params)
-            )
+        data_source_svc = DataSourceService(metadata)
+        response: dict = data_source_svc.get(params)
+        return self.dict_to_message(response)
 
     def list(self, request, context):
         params, metadata = self.parse_request(request, context)
-
-        with self.locator.get_service(
-            "DataSourceService", metadata
-        ) as data_source_service:
-            data_source_vos, total_count = data_source_service.list(params)
-            return self.locator.get_info(
-                "DataSourcesInfo",
-                data_source_vos,
-                total_count,
-                minimal=self.get_minimal(params),
-            )
+        data_source_svc = DataSourceService(metadata)
+        response: dict = data_source_svc.list(params)
+        return self.dict_to_message(response)
 
     def stat(self, request, context):
         params, metadata = self.parse_request(request, context)

src/spaceone/cost_analysis/manager/data_source_manager.py

@@ -80,7 +80,7 @@ def deregister_data_source_by_vo(data_source_vo):
 
     def get_data_source(
         self, data_source_id: str, domain_id: str, workspace_id: str = None
-    ):
+    ) -> DataSource:
         conditions = {"data_source_id": data_source_id, "domain_id": domain_id}
 
         if workspace_id:

src/spaceone/cost_analysis/model/data_source/request.py

@@ -1,16 +1,63 @@
 from typing import Union, Literal
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 __all__ = [
+    "DataSourceRegisterRequest",
     "DataSourceUpdatePermissionsRequest",
+    "DataSourceGetRequest",
+    "DataSourceSearchQueryRequest",
 ]
-State = Literal["ENABLED", "DISABLED"]
+
 DataSourceType = Literal["LOCAL", "EXTERNAL"]
+State = Literal["ENABLED", "DISABLED"]
 SecretType = Literal["MANUAL", "USE_SERVICE_ACCOUNT_SECRET"]
 ResourceGroup = Literal["DOMAIN", "WORKSPACE"]
 
 
+class Plugin(BaseModel):
+    plugin_id: str
+    version: Union[str, None] = None
+    options: Union[dict, None] = None
+    metadata: Union[dict, None] = None
+    secret_data: Union[dict, None] = None
+    schema_id: Union[str, None] = Field(None, alias="schema")
+    secret_id: Union[str, None] = None
+    upgrade_mode: Union[str, None] = None
+
+
+class DataSourceRegisterRequest(BaseModel):
+    name: str
+    data_source_type: DataSourceType
+    provider: Union[str, None] = None
+    secret_type: Union[SecretType, str, None] = None
+    secret_filter: Union[dict, None] = None
+    template: Union[dict, None] = None
+    plugin_info: Union[Plugin, None] = None
+    tags: Union[dict, None] = None
+    resource_group: Union[ResourceGroup, None] = None
+    workspace_id: Union[str, None] = None
+    domain_id: str
+
+
 class DataSourceUpdatePermissionsRequest(BaseModel):
     data_source_id: str
     permissions: dict
     domain_id: str
+
+
+class DataSourceGetRequest(BaseModel):
+    data_source_id: str
+    workspace_id: Union[list, str, None] = None
+    domain_id: str
+
+
+class DataSourceSearchQueryRequest(BaseModel):
+    query: Union[dict, None] = None
+    data_source_id: Union[str, None] = None
+    name: Union[str, None] = None
+    state: Union[State, None] = None
+    data_source_type: Union[DataSourceType, None] = None
+    provider: Union[str, None] = None
+    connected_workspace_id: Union[str, None] = None
+    workspace_id: Union[list, str, None] = None
+    domain_id: str

src/spaceone/cost_analysis/model/data_source/response.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import Union, Literal
+from typing import Union, Literal, List
 from pydantic import BaseModel
 
 from spaceone.core import utils
@@ -13,6 +13,7 @@
 
 __all__ = [
     "DataSourceResponse",
+    "DataSourcesResponse",
 ]
 
 
@@ -48,3 +49,8 @@ def dict(self, *args, **kwargs):
             data.get("last_synchronized_at")
         )
         return data
+
+
+class DataSourcesResponse(BaseModel):
+    results: List[DataSourceResponse]
+    total_count: int

src/spaceone/cost_analysis/service/data_source_service.py

@@ -5,10 +5,8 @@
 from mongoengine import QuerySet
 from spaceone.core.service import *
 from spaceone.cost_analysis.error import *
-from spaceone.cost_analysis.model.data_source.request import (
-    DataSourceUpdatePermissionsRequest,
-)
-from spaceone.cost_analysis.model.data_source.response import DataSourceResponse
+from spaceone.cost_analysis.model.data_source.request import *
+from spaceone.cost_analysis.model.data_source.response import *
 from spaceone.cost_analysis.service.job_service import JobService
 from spaceone.cost_analysis.manager.repository_manager import RepositoryManager
 from spaceone.cost_analysis.manager.secret_manager import SecretManager
@@ -48,8 +46,10 @@ def __init__(self, *args, **kwargs):
         permission="cost-analysis:DataSource.write",
         role_types=["DOMAIN_ADMIN", "WORKSPACE_OWNER"],
     )
-    @check_required(["name", "data_source_type", "domain_id"])
-    def register(self, params):
+    @convert_model
+    def register(
+            self, params: DataSourceRegisterRequest
+    ) -> Union[DataSourceResponse, dict]:
         """Register data source
 
         Args:
@@ -70,6 +70,7 @@ def register(self, params):
         Returns:
             data_source_vo (object)
         """
+        params = params.dict(exclude_unset=True)
 
         domain_id = params["domain_id"]
         data_source_type = params["data_source_type"]
@@ -177,7 +178,7 @@ def register(self, params):
                     data_source_vo
                 )
 
-        return data_source_vo
+        return DataSourceResponse(**data_source_vo.to_dict())
 
     @transaction(
         permission="cost-analysis:DataSource.write",
@@ -228,7 +229,7 @@ def update(self, params):
     )
     @convert_model
     def update_permissions(
-        self, params: DataSourceUpdatePermissionsRequest
+            self, params: DataSourceUpdatePermissionsRequest
     ) -> Union[DataSourceResponse, dict]:
         """Update data source permissions
 
@@ -567,35 +568,45 @@ def sync(self, params):
         permission="cost-analysis:DataSource.read",
         role_types=["DOMAIN_ADMIN", "WORKSPACE_OWNER", "WORKSPACE_MEMBER"],
     )
-    @check_required(["data_source_id", "domain_id"])
-    def get(self, params):
+    @change_value_by_rule("APPEND", "workspace_id", "*")
+    @convert_model
+    def get(self, params: DataSourceGetRequest) -> Union[DataSourceResponse, dict]:
         """Get data source
 
         Args:
             params (dict): {
                 'data_source_id': 'str',  # required
-                'workspace_id': 'str'
+                'workspace_id': 'list'
                 'domain_id': 'str',      # injected from auth
             }
 
         Returns:
             data_source_vo (object)
         """
 
-        data_source_id = params["data_source_id"]
-        domain_id = params["domain_id"]
-        workspace_id = params.get("workspace_id")
+        data_source_id = params.data_source_id
+        domain_id = params.domain_id
+        workspace_id = params.workspace_id
 
-        return self.data_source_mgr.get_data_source(
+        data_source_vo = self.data_source_mgr.get_data_source(
             data_source_id, domain_id, workspace_id
         )
 
+        # Check data fields permissions
+        if self.transaction.get_meta("authorization.role_type") != "DOMAIN_ADMIN":
+            data_source_vo = (
+                self._filter_cost_data_keys_with_permissions_by_data_source_vo(
+                    data_source_vo
+                )
+            )
+
+        return DataSourceResponse(**data_source_vo.to_dict())
+
     @transaction(
         permission="cost-analysis:DataSource.read",
         role_types=["DOMAIN_ADMIN", "WORKSPACE_OWNER", "WORKSPACE_MEMBER"],
     )
     @change_value_by_rule("APPEND", "workspace_id", "*")
-    @check_required(["domain_id"])
     @append_query_filter(
         [
             "data_source_id",
@@ -609,11 +620,15 @@ def get(self, params):
     )
     @change_tag_filter("tags")
     @append_keyword_filter(["data_source_id", "name"])
-    def list(self, params):
+    @convert_model
+    def list(
+            self, params: DataSourceSearchQueryRequest
+    ) -> Union[DataSourcesResponse, dict]:
         """List data sources
 
         Args:
             params (dict): {
+                'query': 'dict (spaceone.api.core.v1.Query)'
                 'data_source_id': 'str',
                 'name': 'str',
                 'state': 'str',
@@ -622,16 +637,19 @@ def list(self, params):
                 'connected_workspace_id': str,
                 'workspace_id': 'list,
                 'domain_id': 'str',
-                'query': 'dict (spaceone.api.core.v1.Query)'
+
             }
 
         Returns:
             data_source_vos (object)
             total_count
         """
 
-        query = params.get("query", {})
-        connected_workspace_id = params.get("connected_workspace_id")
+        query = params.query or {}
+        connected_workspace_id = params.connected_workspace_id
+
+        if self.transaction.get_meta("authorization.role_type") != "DOMAIN_ADMIN":
+            self._check_only_fields_for_permissions(query)
 
         if connected_workspace_id:
             (
@@ -643,7 +661,9 @@ def list(self, params):
         else:
             data_source_vos, total_count = self.data_source_mgr.list_data_sources(query)
 
-        return data_source_vos, total_count
+        data_sources_info = self._get_data_sources_info_by_role_type(data_source_vos)
+
+        return DataSourcesResponse(results=data_sources_info, total_count=total_count)
 
     @transaction(
         permission="cost-analysis:DataSource.read",
@@ -741,21 +761,21 @@ def _get_secret_data(self, secret_id, domain_id):
         return secret_data
 
     @staticmethod
-    def _validate_plugin_info(plugin_info, secret_type):
+    def _validate_plugin_info(plugin_info: dict, secret_type: str) -> None:
         if "plugin_id" not in plugin_info:
             raise ERROR_REQUIRED_PARAMETER(key="plugin_info.plugin_id")
 
         if (
-            plugin_info.get("upgrade_mode", "AUTO") == "MANUAL"
-            and "version" not in plugin_info
+                plugin_info.get("upgrade_mode", "AUTO") == "MANUAL"
+                and "version" not in plugin_info
         ):
             raise ERROR_REQUIRED_PARAMETER(key="plugin_info.version")
 
         if secret_type == "MANUAL" and plugin_info.get("secret_data") is None:
             raise ERROR_REQUIRED_PARAMETER(key="plugin_info.secret_data")
 
     def create_data_source_account_with_data_source_vo(
-        self, accounts_info: dict, data_source_vo: DataSource
+            self, accounts_info: dict, data_source_vo: DataSource
     ) -> None:
         data_source_id = data_source_vo.data_source_id
         workspace_id = data_source_vo.workspace_id
@@ -813,7 +833,7 @@ def create_data_source_account_with_data_source_vo(
             )
 
     def _get_data_source_account_vo_map(
-        self, data_source_id: str, domain_id: str
+            self, data_source_id: str, domain_id: str
     ) -> dict:
         data_source_account_vo_map = {}
         data_source_account_vos = (
@@ -828,7 +848,7 @@ def _get_data_source_account_vo_map(
         return data_source_account_vo_map
 
     def _change_filter_connected_workspace_data_source(
-        self, query: dict, connected_workspace_id: str
+            self, query: dict, connected_workspace_id: str
     ) -> Tuple[Union[QuerySet, list], int]:
         connected_data_source_ids = []
         domain_id = self._get_domain_id_from_filter(query)
@@ -873,6 +893,22 @@ def _change_filter_connected_workspace_data_source(
 
         return self.data_source_mgr.list_data_sources(query)
 
+    def _get_data_sources_info_by_role_type(self, data_source_vos: list) -> list:
+        data_sources_info = []
+        if self.transaction.get_meta("authorization.role_type") != "DOMAIN_ADMIN":
+            for data_source_vo in data_source_vos:
+                data_source_vo = (
+                    self._filter_cost_data_keys_with_permissions_by_data_source_vo(
+                        data_source_vo
+                    )
+                )
+                data_sources_info.append(data_source_vo.to_dict())
+        else:
+            for data_source_vo in data_source_vos:
+                data_sources_info.append(data_source_vo.to_dict())
+
+        return data_sources_info
+
     @staticmethod
     def _get_domain_id_from_filter(query: dict) -> str:
         for condition in query.get("filter", []):
@@ -903,3 +939,29 @@ def _get_copied_remove_only_filter_query(query: dict) -> dict:
         copied_query.pop("minimal", None)
         _LOGGER.debug(f"[_get_copied_remove_only_filter_query] query: {copied_query}")
         return copied_query
+
+    @staticmethod
+    def _filter_cost_data_keys_with_permissions_by_data_source_vo(
+            data_source_vo: DataSource,
+    ) -> DataSource:
+        if data_source_vo.permissions:
+            deny = data_source_vo.permissions.get("deny", [])
+            cost_data_keys = data_source_vo.cost_data_keys or []
+            for deny_key in deny:
+                if deny_key.startswith("data."):
+                    split_denied_key = deny_key.split(".")[1]
+                    if split_denied_key in cost_data_keys:
+                        cost_data_keys.remove(split_denied_key)
+            data_source_vo.cost_data_keys = cost_data_keys
+        return data_source_vo
+
+    @staticmethod
+    def _check_only_fields_for_permissions(query: dict) -> None:
+        only_fields = query.get("only")
+        for only_field in only_fields:
+            if only_field == "cost_data_keys":
+                if "permissions" not in only_fields:
+                    raise ERROR_INVALID_PARAMETER(
+                        key="permissions",
+                        reason="when you want to get 'cost_data_keys', you must include 'permissions' field in 'only' field.",
+                    )