Kasiah/community dot com connector

parsons/__init__.py

@@ -45,6 +45,7 @@
     ("parsons.catalist.catalist", "CatalistMatch"),
     ("parsons.census.census", "Census"),
     ("parsons.civis.civisclient", "CivisClient"),
+    ("parsons.community.community", "Community"),
     ("parsons.controlshift.controlshift", "Controlshift"),
     ("parsons.copper.copper", "Copper"),
     ("parsons.crowdtangle.crowdtangle", "CrowdTangle"),

parsons/community/__init__.py

@@ -0,0 +1,3 @@
+from parsons.community.community import Community
+
+__all__ = ["Community"]

parsons/community/community.py

@@ -0,0 +1,101 @@
+import logging
+from parsons.utilities.api_connector import APIConnector
+from parsons.utilities import check_env
+from parsons.etl import Table
+
+logger = logging.getLogger(__name__)
+
+COMMUNITY_API_ENDPOINT = "https://dl.community.com/download/v1/files/"
+
+
+class Community(object):
+    """
+    Instantiate class.
+
+       `Args:`
+            community_client_id: str
+                The Community provided Client ID. Not required if ``COMMUNITY_CLIENT_ID`` env
+                variable set.
+            community_access_token: str
+                The Community provided access token. Not required if ``COMMUNITY_ACCESS_TOKEN`` env
+                variable set.
+            community_uri: str
+                The URI to access the  API. Not required, default is
+                https://dl.community.com/download/v1/files/. You can set an ``COMMUNITY_URL`` env
+                variable or use this URI parameter if a different endpoint is necessary.
+
+    API Documentation: https://developer.community.com/reference/data-export-api-downloading-data
+    """
+
+    def __init__(self, community_client_id=None, community_access_token=None, community_url=None):
+        self.community_client_id = check_env.check("community_client_id", community_client_id)
+        self.community_access_token = check_env.check(
+            "community_access_token", community_access_token
+        )
+        self.uri = (
+            check_env.check("COMMUNITY_URL", community_url, optional=True)
+            or f"{COMMUNITY_API_ENDPOINT}/{community_client_id}/"
+        )
+        self.headers = {
+            "Authorization": f"Bearer {self.community_access_token}",
+        }
+        self.client = APIConnector(
+            self.uri,
+            headers=self.headers,
+        )
+
+    def get_request(self, filename):
+        """
+        GET request to Community.com API to get the CSV data.
+
+        `Args:`
+            filename: str
+                Data filename you are requesting.
+                Options:
+                    'campaigns': Campaign Performance data
+                    'outbound_message_type_usage`: Message Segment Usage data
+                    'campaign_links': Campaign Link Performance data
+                    'members': Member Details data
+                    'member_state_changes': Member Subscription Status data
+                    'custom_member_data': Custom Member Data
+                    'communities': Communities data
+                    'member_communities': Member Communities data
+
+        `Returns:`
+            Response of GET request; a successful response returns the CSV formatted data
+        """
+
+        logger.info(f"Requesting {filename}")
+        url = (
+            f"{filename}.csv.gz"
+            if filename != "outbound_message_type_usage"
+            else f"{filename}.csv.gz/segment-based-subscription"
+        )
+        response = self.client.get_request(url=url, return_format="content")
+        return response
+
+    def get_data_export(self, filename):
+        """
+        Get specified data from Community.com API as Parsons table.
+
+        `Args:`
+            filename: str
+                Data filename you are requesting.
+                Options:
+                    'campaigns': Campaign Performance data
+                    'outbound_message_type_usage`: Message Segment Usage data
+                    'campaign_links': Campaign Link Performance data
+                    'members': Member Details data
+                    'member_state_changes': Member Subscription Status data
+                    'custom_member_data': Custom Member Data
+                    'communities': Communities data
+                    'member_communities': Member Communities data
+
+        `Returns:`
+            Contents of the generated contribution CSV as a Parsons table.
+        """
+
+        get_request_response = self.get_request(filename=filename)
+        response_string = get_request_response.decode("utf-8")
+        table = Table.from_csv_string(response_string)
+        return table

parsons/google/google_cloud_storage.py

@@ -47,9 +47,7 @@ class GoogleCloudStorage(object):
 
     def __init__(self, app_creds=None, project=None):
         env_credentials_path = str(uuid.uuid4())
-        setup_google_application_credentials(
-            app_creds, target_env_var_name=env_credentials_path
-        )
+        setup_google_application_credentials(app_creds, target_env_var_name=env_credentials_path)
         credentials = load_google_application_credentials(env_credentials_path)
         self.project = project
 
@@ -297,9 +295,7 @@ def delete_blob(self, bucket_name, blob_name):
         blob.delete()
         logger.info(f"{blob_name} blob in {bucket_name} bucket deleted.")
 
-    def upload_table(
-        self, table, bucket_name, blob_name, data_type="csv", default_acl=None
-    ):
+    def upload_table(self, table, bucket_name, blob_name, data_type="csv", default_acl=None):
         """
         Load the data from a Parsons table into a blob.
 
@@ -327,17 +323,18 @@ def upload_table(
             # CSVView. Once any transformations are made, the Table.table
             # becomes a different petl class
             if isinstance(table.table, petl.io.csv_py3.CSVView):
-                local_file = table.table.source.filename
+                try:
+                    local_file = table.table.source.filename
+                except AttributeError:
+                    local_file = table.to_csv()
             else:
                 local_file = table.to_csv()
             content_type = "text/csv"
         elif data_type == "json":
             local_file = table.to_json()
             content_type = "application/json"
         else:
-            raise ValueError(
-                f"Unknown data_type value ({data_type}): must be one of: csv or json"
-            )
+            raise ValueError(f"Unknown data_type value ({data_type}): must be one of: csv or json")
 
         try:
             blob.upload_from_filename(
@@ -407,9 +404,7 @@ def copy_bucket_to_gcs(
                 Secret key to authenticate storage transfer
         """
         if source not in ["gcs", "s3"]:
-            raise ValueError(
-                f"Blob transfer only supports gcs and s3 sources [source={source}]"
-            )
+            raise ValueError(f"Blob transfer only supports gcs and s3 sources [source={source}]")
         if source_path and source_path[-1] != "/":
             raise ValueError("Source path much end in a '/'")
 
@@ -596,13 +591,9 @@ def unzip_blob(
         }
 
         file_extension = compression_params[compression_type]["file_extension"]
-        compression_function = compression_params[compression_type][
-            "compression_function"
-        ]
+        compression_function = compression_params[compression_type]["compression_function"]
 
-        compressed_filepath = self.download_blob(
-            bucket_name=bucket_name, blob_name=blob_name
-        )
+        compressed_filepath = self.download_blob(bucket_name=bucket_name, blob_name=blob_name)
 
         decompressed_filepath = compressed_filepath.replace(file_extension, "")
         decompressed_blob_name = (
@@ -634,9 +625,7 @@ def __gzip_decompress_and_write_to_gcs(self, **kwargs):
         bucket_name = kwargs.pop("bucket_name")
 
         with gzip.open(compressed_filepath, "rb") as f_in:
-            logger.debug(
-                f"Uploading uncompressed file to GCS: {decompressed_blob_name}"
-            )
+            logger.debug(f"Uploading uncompressed file to GCS: {decompressed_blob_name}")
             bucket = self.get_bucket(bucket_name=bucket_name)
             blob = storage.Blob(name=decompressed_blob_name, bucket=bucket)
             blob.upload_from_file(file_obj=f_in, rewind=True, timeout=3600)
@@ -656,9 +645,7 @@ def __zip_decompress_and_write_to_gcs(self, **kwargs):
         with zipfile.ZipFile(compressed_filepath) as path_:
             # Open the underlying file
             with path_.open(decompressed_blob_in_archive) as f_in:
-                logger.debug(
-                    f"Uploading uncompressed file to GCS: {decompressed_blob_name}"
-                )
+                logger.debug(f"Uploading uncompressed file to GCS: {decompressed_blob_name}")
                 bucket = self.get_bucket(bucket_name=bucket_name)
                 blob = storage.Blob(name=decompressed_blob_name, bucket=bucket)
                 blob.upload_from_file(file_obj=f_in, rewind=True, timeout=3600)

parsons/utilities/api_connector.py

@@ -81,7 +81,7 @@ def request(self, url, req_type, json=None, data=None, params=None):
             params=params,
         )
 
-    def get_request(self, url, params=None):
+    def get_request(self, url, params=None, return_format="json"):
         """
         Make a GET request.
 
@@ -96,9 +96,14 @@ def get_request(self, url, params=None):
 
         r = self.request(url, "GET", params=params)
         self.validate_response(r)
-        logger.debug(r.json())
 
-        return r.json()
+        if return_format == "json":
+            logger.debug(r.json())
+            return r.json()
+        elif return_format == "content":
+            return r.content
+        else:
+            raise RuntimeError(f"{return_format} is not a valid format, change to json or content")
 
     def post_request(
         self, url, params=None, data=None, json=None, success_codes=[200, 201, 202, 204]

test/test_community/test_community.py

@@ -0,0 +1,40 @@
+import unittest
+import requests_mock
+from parsons import Community
+
+
+TEST_CLIENT_ID = "someuuid"
+TEST_CLIENT_TOKEN = "somesecret"
+
+TEST_FILENAME = "campaigns"
+TEST_URI = f"https://faketestingurl.com/{TEST_CLIENT_ID}"
+TEST_FULL_URL = f"{TEST_URI}/{TEST_FILENAME}.csv.gz"
+
+TEST_GET_RESPONSE_CSV_STRING = b'"CAMPAIGN_ID","LEADER_ID"\n"0288","6e83b"\n'
+
+TEST_EXPECTED_COLUMNS = [
+    "CAMPAIGN_ID",
+    "LEADER_ID",
+]
+
+
+class TestCommunity(unittest.TestCase):
+    @requests_mock.Mocker()
+    def setUp(self, m):
+        self.com = Community(TEST_CLIENT_ID, TEST_CLIENT_TOKEN, TEST_URI)
+
+    @requests_mock.Mocker()
+    def test_successful_get_request(self, m):
+        m.get(TEST_FULL_URL, content=TEST_GET_RESPONSE_CSV_STRING)
+
+        assert self.com.get_request(filename=TEST_FILENAME) == TEST_GET_RESPONSE_CSV_STRING
+
+    # test get resource
+    @requests_mock.Mocker()
+    def test_successful_get_data_export(self, m):
+        m.get(TEST_FULL_URL, content=TEST_GET_RESPONSE_CSV_STRING)
+
+        table = self.com.get_data_export(
+            TEST_FILENAME,
+        )
+        assert TEST_EXPECTED_COLUMNS == table.columns