From 2f402fc2bfb8bcc43799395990f012eeec3fa820 Mon Sep 17 00:00:00 2001 From: Jordan Scott Date: Thu, 19 May 2022 13:56:10 -0500 Subject: [PATCH 01/21] added linkedin pages source connector --- .../source-linkedin-pages/.dockerignore | 6 + .../source-linkedin-pages/Dockerfile | 38 + .../source-linkedin-pages/README.md | 132 ++ .../acceptance-test-config.yml | 30 + .../acceptance-test-docker.sh | 16 + .../source-linkedin-pages/bootstrap.md | 7 + .../source-linkedin-pages/build.gradle | 9 + .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 23 + .../integration_tests/acceptance.py | 13 + .../integration_tests/configured_catalog.json | 67 + .../integration_tests/invalid_config.json | 7 + .../integration_tests/sample_config.json | 7 + .../integration_tests/sample_state.json | 5 + .../connectors/source-linkedin-pages/main.py | 12 + .../source-linkedin-pages/requirements.txt | 2 + .../connectors/source-linkedin-pages/setup.py | 30 + .../source_linkedin_pages/__init__.py | 8 + .../source_linkedin_pages/analytics.py | 185 ++ .../schemas/follower_statistics.json | 201 ++ .../schemas/organization_lookup.json | 272 +++ .../schemas/page_statistics.json | 1726 +++++++++++++++++ .../schemas/share_statistics.json | 68 + .../source_linkedin_pages/schemas/shares.json | 109 ++ .../schemas/total_follower_count.json | 8 + .../schemas/ugc_posts.json | 302 +++ .../source_linkedin_pages/source.py | 166 ++ .../source_linkedin_pages/spec.json | 78 + .../source_linkedin_pages/utils.py | 324 ++++ .../unit_tests/__init__.py | 3 + .../unit_tests/test_incremental_streams.py | 59 + .../unit_tests/test_source.py | 22 + .../unit_tests/test_streams.py | 83 + 33 files changed, 4021 insertions(+) create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/.dockerignore create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/Dockerfile create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/README.md create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/bootstrap.md create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/build.gradle create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/main.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/requirements.txt create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/setup.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/__init__.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/analytics.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_incremental_streams.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-linkedin-pages/.dockerignore b/airbyte-integrations/connectors/source-linkedin-pages/.dockerignore new file mode 100644 index 0000000000000..53de6536d1ff3 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_linkedin_pages +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile b/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile new file mode 100644 index 0000000000000..6eff6390b8787 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.7.11-alpine3.14 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_linkedin_pages ./source_linkedin_pages + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-linkedin-pages diff --git a/airbyte-integrations/connectors/source-linkedin-pages/README.md b/airbyte-integrations/connectors/source-linkedin-pages/README.md new file mode 100644 index 0000000000000..5671f959d3ff5 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/README.md @@ -0,0 +1,132 @@ +# Linkedin Pages Source + +This is the repository for the Linkedin Pages source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/linkedin-pages). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-linkedin-pages:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/linkedin-pages) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_linkedin_pages/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source linkedin-pages test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-linkedin-pages:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-linkedin-pages:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-linkedin-pages:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-pages:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-pages:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-linkedin-pages:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-linkedin-pages:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-linkedin-pages:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml new file mode 100644 index 0000000000000..15191f1b4d4d6 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-linkedin-pages:dev0.1.1 +tests: + spec: + - spec_path: "source_linkedin_pages/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + # incremental: # TODO if your connector does not implement incremental sync, remove this block + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-linkedin-pages/bootstrap.md b/airbyte-integrations/connectors/source-linkedin-pages/bootstrap.md new file mode 100644 index 0000000000000..90e5cbd5eb5c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/bootstrap.md @@ -0,0 +1,7 @@ +The LinkedIn Marketing Developer Platform API can be used to pull data from LinkedIn Organizations such as company page details, follower counts, follower statistics, and all sorts of organic content data. + +You must have a LinkedIn Developers' App created in order to request access to the Marketing Developer Platform API. API Access approval takes up to 72 hours after submitting a ~15-question form. + +The app also must be verified by an admin of the LinkedIn organization your app is created for. Once the app is "verified" and granted access to the Marketing Developer Platform API, you can use their easy-peasy OAuth Token Tools to generate access tokens **and** refresh tokens. + +You can access the `client id` and `client secret` in the **Auth** tab of the app dashboard to round out all of the authorization needs you may have. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/build.gradle b/airbyte-integrations/connectors/source-linkedin-pages/build.gradle new file mode 100644 index 0000000000000..df9098d6060fc --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_linkedin_pages' +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/__init__.py b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..ceae7331d2d9a --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json @@ -0,0 +1,23 @@ +{ + "organization_lookup": { + "lastModified": "2050-01-01" + }, + "follower_statistics": { + "lastModified": "2050-01-01" + }, + "page_statistics": { + "lastModified": "2050-01-01" + }, + "share_statistics": { + "lastModified": "2050-01-01" + }, + "shares": { + "lastModified": "2050-01-01" + }, + "total_follower_count": { + "end_date": "2050-01-01" + }, + "ugc_posts": { + "end_date": "2050-01-01" + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py new file mode 100644 index 0000000000000..1b85ccdcde521 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + yield \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..6d5c31ddc7549 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json @@ -0,0 +1,67 @@ +{ + "streams": [ + { + "stream": { + "name": "organization_lookup", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "follower_statistics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "page_statistics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "share_statistics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "shares", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "total_follower_count", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "ugc_posts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..5c8c7b29a99f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/invalid_config.json @@ -0,0 +1,7 @@ +{ + "org_id": 12345678, + "credentials": { + "auth_method": "access_token", + "access_token": "wrong_token_sra6ibiw0ZWEdMnC0ZizeD1gLRQP6u1pkQl" + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json new file mode 100644 index 0000000000000..c06b53f2c96e3 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json @@ -0,0 +1,7 @@ +{ + "org_id": 12345678, + "credentials": { + "auth_method": "access_token", + "access_token": "drAL4JXBJ6v1qaU9iWargosra6ibiw0ZWEdMnC0ZizeD1gLRQP6u1pkQ" + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_state.json new file mode 100644 index 0000000000000..3587e579822d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/main.py b/airbyte-integrations/connectors/source-linkedin-pages/main.py new file mode 100644 index 0000000000000..d601480ca89df --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/main.py @@ -0,0 +1,12 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import sys + +from airbyte_cdk.entrypoint import launch +from source_linkedin_pages import SourceLinkedinPages + +if __name__ == "__main__": + source = SourceLinkedinPages() + launch(source, sys.argv[1:]) \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/requirements.txt b/airbyte-integrations/connectors/source-linkedin-pages/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-linkedin-pages/setup.py b/airbyte-integrations/connectors/source-linkedin-pages/setup.py new file mode 100644 index 0000000000000..bae2e6204f394 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/setup.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", + "pendulum~=2.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_linkedin_pages", + description="Source implementation for Linkedin Company Pages.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/__init__.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/__init__.py new file mode 100644 index 0000000000000..e4157287bd16d --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceLinkedinPages + +__all__ = ["SourceLinkedinPages"] diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/analytics.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/analytics.py new file mode 100644 index 0000000000000..9cbdc2d307919 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/analytics.py @@ -0,0 +1,185 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from collections import defaultdict +from typing import Any, Iterable, List, Mapping + +import pendulum as pdm + +from .utils import get_parent_stream_values + +# LinkedIn has a max of 20 fields per request. We make chunks by size of 17 fields +# to have the `dateRange`, `pivot`, and `pivotValue` be included as well. +FIELDS_CHUNK_SIZE = 17 +# Number of days ahead for date slices, from start date. +WINDOW_IN_DAYS = 30 +# List of adAnalyticsV2 fields available for fetch +ANALYTICS_FIELDS_V2: List = [ + "actionClicks", + "adUnitClicks", + "approximateUniqueImpressions", + "cardClicks", + "cardImpressions", + "clicks", + "commentLikes", + "comments", + "companyPageClicks", + "conversionValueInLocalCurrency", + "costInLocalCurrency", + "costInUsd", + "dateRange", + "externalWebsiteConversions", + "externalWebsitePostClickConversions", + "externalWebsitePostViewConversions", + "follows", + "fullScreenPlays", + "impressions", + "landingPageClicks", + "leadGenerationMailContactInfoShares", + "leadGenerationMailInterestedClicks", + "likes", + "oneClickLeadFormOpens", + "oneClickLeads", + "opens", + "otherEngagements", + "pivot", + "pivotValue", + "pivotValues", + "reactions", + "sends", + "shares", + "textUrlClicks", + "totalEngagements", + "videoCompletions", + "videoFirstQuartileCompletions", + "videoMidpointCompletions", + "videoStarts", + "videoThirdQuartileCompletions", + "videoViews", + "viralCardClicks", + "viralCardImpressions", + "viralClicks", + "viralCommentLikes", + "viralComments", + "viralCompanyPageClicks", + "viralExternalWebsiteConversions", + "viralExternalWebsitePostClickConversions", + "viralExternalWebsitePostViewConversions", + "viralFollows", + "viralFullScreenPlays", + "viralImpressions", + "viralLandingPageClicks", + "viralLikes", + "viralOneClickLeadFormOpens", + "viralOneClickLeads", + "viralOtherEngagements", + "viralReactions", + "viralShares", + "viralTotalEngagements", + "viralVideoCompletions", + "viralVideoFirstQuartileCompletions", + "viralVideoMidpointCompletions", + "viralVideoStarts", + "viralVideoThirdQuartileCompletions", + "viralVideoViews", +] + +# Fields that are always present in fields_set chunks +BASE_ANALLYTICS_FIELDS = ["dateRange", "pivot", "pivotValue"] + + +def chunk_analytics_fields( + fields: List = ANALYTICS_FIELDS_V2, + base_fields: List = BASE_ANALLYTICS_FIELDS, + fields_chunk_size: int = FIELDS_CHUNK_SIZE, +) -> Iterable[List]: + """ + Chunks the list of available fields into the chunks of equal size. + """ + # Make chunks + chunks = list((fields[f : f + fields_chunk_size] for f in range(0, len(fields), fields_chunk_size))) + # Make sure base_fields are within the chunks + for chunk in chunks: + for field in base_fields: + if field not in chunk: + chunk.append(field) + yield from chunks + + +def make_date_slices(start_date: str, end_date: str = None, window_in_days: int = WINDOW_IN_DAYS) -> Iterable[List]: + """ + Produces date slices from start_date to end_date (if specified), + otherwise end_date will be present time. + """ + start = pdm.parse(start_date) + end = pdm.parse(end_date) if end_date else pdm.now() + date_slices = [] + while start < end: + slice_end_date = start.add(days=window_in_days) + date_slice = { + "start.day": start.day, + "start.month": start.month, + "start.year": start.year, + "end.day": slice_end_date.day, + "end.month": slice_end_date.month, + "end.year": slice_end_date.year, + } + date_slices.append({"dateRange": date_slice}) + start = slice_end_date + yield from date_slices + + +def make_analytics_slices( + record: Mapping[str, Any], key_value_map: Mapping[str, Any], start_date: str, end_date: str = None +) -> Iterable[Mapping[str, Any]]: + """ + We drive the ability to directly pass the prepared parameters inside the stream_slice. + The output of this method is ready slices for analytics streams: + """ + # define the base_slice + base_slice = get_parent_stream_values(record, key_value_map) + # add chunked fields, date_slices to the base_slice + analytics_slices = [] + for fields_set in chunk_analytics_fields(): + base_slice["fields"] = ",".join(map(str, fields_set)) + for date_slice in make_date_slices(start_date, end_date): + base_slice.update(**date_slice) + analytics_slices.append(base_slice.copy()) + yield from analytics_slices + + +def update_analytics_params(stream_slice: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Produces the date range parameters from input stream_slice + """ + return { + # Start date range + "dateRange.start.day": stream_slice["dateRange"]["start.day"], + "dateRange.start.month": stream_slice["dateRange"]["start.month"], + "dateRange.start.year": stream_slice["dateRange"]["start.year"], + # End date range + "dateRange.end.day": stream_slice["dateRange"]["end.day"], + "dateRange.end.month": stream_slice["dateRange"]["end.month"], + "dateRange.end.year": stream_slice["dateRange"]["end.year"], + # Chunk of fields + "fields": stream_slice["fields"], + } + + +def merge_chunks(chunked_result: Iterable[Mapping[str, Any]], merge_by_key: str) -> Iterable[Mapping[str, Any]]: + """ + We need to merge the chunked API responses + into the single structure using any available unique field. + """ + # Merge the pieces together + merged = defaultdict(dict) + for chunk in chunked_result: + for item in chunk: + merged[item[merge_by_key]].update(item) + # Clean up the result by getting out the values of the merged keys + result = [] + for item in merged: + result.append(merged.get(item)) + yield from result \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json new file mode 100644 index 0000000000000..1e1d0298f3d93 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -0,0 +1,201 @@ +{ + "type": "object", + "properties": { + "paging": { + "type": "object", + "properties": { + "start": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "links": { + "type": "array", + "items": {} + } + } + }, + "elements": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "followerCountsByAssociationType": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "followerCounts": { + "type": "object", + "properties": { + "organicFollowerCount": { + "type": "integer" + }, + "paidFollowerCount": { + "type": "integer" + } + } + }, + "associationType": { + "type": "string" + } + } + } + ] + }, + "followerCountsByRegion": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "region": { + "type": "string" + }, + "followerCounts": { + "type": "object", + "properties": { + "organicFollowerCount": { + "type": "integer" + }, + "paidFollowerCount": { + "type": "integer" + } + } + } + } + } + ] + }, + "followerCountsBySeniority": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "followerCounts": { + "type": "object", + "properties": { + "organicFollowerCount": { + "type": "integer" + }, + "paidFollowerCount": { + "type": "integer" + } + } + }, + "seniority": { + "type": "string" + } + } + } + ] + }, + "followerCountsByIndustry": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "followerCounts": { + "type": "object", + "properties": { + "organicFollowerCount": { + "type": "integer" + }, + "paidFollowerCount": { + "type": "integer" + } + } + }, + "industry": { + "type": "string" + } + } + } + ] + }, + "followerCountsByFunction": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "followerCounts": { + "type": "object", + "properties": { + "organicFollowerCount": { + "type": "integer" + }, + "paidFollowerCount": { + "type": "integer" + } + } + }, + "function": { + "type": "string" + } + } + } + ] + }, + "followerCountsByStaffCountRange": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "followerCounts": { + "type": "object", + "properties": { + "organicFollowerCount": { + "type": "integer" + }, + "paidFollowerCount": { + "type": "integer" + } + } + }, + "staffCountRange": { + "type": "string" + } + } + } + ] + }, + "followerCountsByCountry": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "followerCounts": { + "type": "object", + "properties": { + "organicFollowerCount": { + "type": "integer" + }, + "paidFollowerCount": { + "type": "integer" + } + } + }, + "country": { + "type": "string" + } + } + } + ] + }, + "organizationalEntity": { + "type": "string" + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json new file mode 100644 index 0000000000000..11e1446820854 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -0,0 +1,272 @@ +{ + "type": "object", + "properties": { + "vanityName": { + "type": "string" + }, + "localizedName": { + "type": "string" + }, + "website": { + "type": "object", + "properties": { + "localized": { + "type": "object", + "properties": { + "en_US": { + "type": "string" + } + } + }, + "preferredLocale": { + "type": "object", + "properties": { + "country": { + "type": "string" + }, + "language": { + "type": "string" + } + } + } + } + }, + "foundedOn": { + "type": "object", + "properties": { + "year": { + "type": "integer" + } + } + }, + "groups": { + "type": "array", + "items": {} + }, + "description": { + "type": "object", + "properties": { + "localized": { + "type": "object", + "properties": { + "en_US": { + "type": "string" + } + } + }, + "preferredLocale": { + "type": "object", + "properties": { + "country": { + "type": "string" + }, + "language": { + "type": "string" + } + } + } + } + }, + "versionTag": { + "type": "string" + }, + "coverPhotoV2": { + "type": "object", + "properties": { + "cropped": { + "type": "string" + }, + "original": { + "type": "string" + }, + "cropInfo": { + "type": "object", + "properties": { + "x": { + "type": "integer" + }, + "width": { + "type": "integer" + }, + "y": { + "type": "integer" + }, + "height": { + "type": "integer" + } + } + } + } + }, + "defaultLocale": { + "type": "object", + "properties": { + "country": { + "type": "string" + }, + "language": { + "type": "string" + } + } + }, + "organizationType": { + "type": "string" + }, + "alternativeNames": { + "type": "array", + "items": {} + }, + "specialties": { + "type": "array", + "items": {} + }, + "staffCountRange": { + "type": "string" + }, + "localizedSpecialties": { + "type": "array", + "items": {} + }, + "industries": { + "type": "array", + "items": [ + { + "type": "string" + } + ] + }, + "name": { + "type": "object", + "properties": { + "localized": { + "type": "object", + "properties": { + "en_US": { + "type": "string" + } + } + }, + "preferredLocale": { + "type": "object", + "properties": { + "country": { + "type": "string" + }, + "language": { + "type": "string" + } + } + } + } + }, + "primaryOrganizationType": { + "type": "string" + }, + "locations": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "locationType": { + "type": "string" + }, + "description": { + "type": "object", + "properties": { + "localized": { + "type": "object", + "properties": { + "en_US": { + "type": "string" + } + } + }, + "preferredLocale": { + "type": "object", + "properties": { + "country": { + "type": "string" + }, + "language": { + "type": "string" + } + } + } + } + }, + "address": { + "type": "object", + "properties": { + "geographicArea": { + "type": "string" + }, + "country": { + "type": "string" + }, + "city": { + "type": "string" + }, + "line1": { + "type": "string" + }, + "postalCode": { + "type": "string" + } + } + }, + "localizedDescription": { + "type": "string" + }, + "geoLocation": { + "type": "string" + }, + "streetAddressFieldState": { + "type": "string" + } + } + } + ] + }, + "id": { + "type": "integer" + }, + "localizedDescription": { + "type": "string" + }, + "$URN": { + "type": "string" + }, + "localizedWebsite": { + "type": "string" + }, + "logoV2": { + "type": "object", + "properties": { + "cropped": { + "type": "string" + }, + "original": { + "type": "string" + }, + "cropInfo": { + "type": "object", + "properties": { + "x": { + "type": "integer" + }, + "width": { + "type": "integer" + }, + "y": { + "type": "integer" + }, + "height": { + "type": "integer" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json new file mode 100644 index 0000000000000..a003e0efc4bc0 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json @@ -0,0 +1,1726 @@ +{ + "type": "object", + "properties": { + "paging": { + "type": "object", + "properties": { + "start": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "links": { + "type": "array", + "items": {} + } + } + }, + "elements": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "pageStatisticsBySeniority": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "pageStatistics": { + "type": "object", + "properties": { + "views": { + "type": "object", + "properties": { + "mobileProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allDesktopPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "insightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allMobilePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "jobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "productsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "peoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "overviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "lifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "careersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopPeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "aboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobilePeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + } + } + } + } + }, + "seniority": { + "type": "string" + } + } + } + ] + }, + "pageStatisticsByCountry": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "country": { + "type": "string" + }, + "pageStatistics": { + "type": "object", + "properties": { + "views": { + "type": "object", + "properties": { + "mobileProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allDesktopPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "insightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allMobilePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "jobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "productsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "peoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "overviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "lifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "careersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopPeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "aboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobilePeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + } + } + } + } + } + } + } + ] + }, + "pageStatisticsByIndustry": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "pageStatistics": { + "type": "object", + "properties": { + "views": { + "type": "object", + "properties": { + "mobileProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allDesktopPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "insightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allMobilePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "jobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "productsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "peoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "overviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "lifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "careersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopPeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "aboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobilePeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + } + } + } + } + } + } + } + ] + }, + "organization": { + "type": "string" + }, + "totalPageStatistics": { + "type": "object", + "properties": { + "clicks": { + "type": "object", + "properties": { + "mobileCareersPageClicks": { + "type": "object", + "properties": { + "careersPageJobsClicks": { + "type": "integer" + }, + "careersPagePromoLinksClicks": { + "type": "integer" + }, + "careersPageEmployeesClicks": { + "type": "integer" + } + } + }, + "careersPageClicks": { + "type": "object", + "properties": { + "careersPagePromoLinksClicks": { + "type": "integer" + }, + "careersPageBannerPromoClicks": { + "type": "integer" + }, + "careersPageJobsClicks": { + "type": "integer" + }, + "careersPageEmployeesClicks": { + "type": "integer" + } + } + } + } + }, + "views": { + "type": "object", + "properties": { + "mobileProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allDesktopPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "insightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allMobilePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "jobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "productsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "peoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "overviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "lifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "careersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopPeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "aboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobilePeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + } + } + } + } + }, + "pageStatisticsByStaffCountRange": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "staffCountRange": { + "type": "string" + }, + "pageStatistics": { + "type": "object", + "properties": { + "views": { + "type": "object", + "properties": { + "mobileProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allDesktopPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "insightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allMobilePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "jobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "productsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "peoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "overviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "lifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "careersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopPeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "aboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobilePeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + } + } + } + } + } + } + } + ] + }, + "pageStatisticsByRegion": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "region": { + "type": "string" + }, + "pageStatistics": { + "type": "object", + "properties": { + "views": { + "type": "object", + "properties": { + "mobileProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allDesktopPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "insightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allMobilePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "jobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "productsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "peoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "overviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "lifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "careersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopPeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "aboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobilePeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + } + } + } + } + } + } + } + ] + }, + "pageStatisticsByFunction": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "pageStatistics": { + "type": "object", + "properties": { + "views": { + "type": "object", + "properties": { + "mobileProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allDesktopPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "insightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allMobilePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "jobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "productsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "peoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "overviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "lifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "careersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopPeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "aboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobilePeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + } + } + } + } + }, + "function": { + "type": "string" + } + } + } + ] + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json new file mode 100644 index 0000000000000..dfc9a7e6a7115 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json @@ -0,0 +1,68 @@ +{ + "type": "object", + "properties": { + "paging": { + "type": "object", + "properties": { + "start": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "links": { + "type": "array", + "items": {} + }, + "total": { + "type": "integer" + } + } + }, + "elements": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "totalShareStatistics": { + "type": "object", + "properties": { + "uniqueImpressionsCount": { + "type": "integer" + }, + "shareCount": { + "type": "integer" + }, + "shareMentionsCount": { + "type": "integer" + }, + "engagement": { + "type": "number" + }, + "clickCount": { + "type": "integer" + }, + "likeCount": { + "type": "integer" + }, + "impressionCount": { + "type": "integer" + }, + "commentMentionsCount": { + "type": "integer" + }, + "commentCount": { + "type": "integer" + } + } + }, + "organizationalEntity": { + "type": "string" + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json new file mode 100644 index 0000000000000..5e0fd75a811ce --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json @@ -0,0 +1,109 @@ +{ + "type": "object", + "properties": { + "paging": { + "type": "object", + "properties": { + "start": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "links": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "rel": { + "type": "string" + }, + "href": { + "type": "string" + } + } + } + ] + }, + "total": { + "type": "integer" + } + } + }, + "elements": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "owner": { + "type": "string" + }, + "activity": { + "type": "string" + }, + "edited": { + "type": "boolean" + }, + "resharedShare": { + "type": "string" + }, + "created": { + "type": "object", + "properties": { + "actor": { + "type": "string" + }, + "time": { + "type": "integer" + } + } + }, + "lastModified": { + "type": "object", + "properties": { + "actor": { + "type": "string" + }, + "time": { + "type": "integer" + } + } + }, + "text": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + }, + "id": { + "type": "string" + }, + "originalShare": { + "type": "string" + }, + "distribution": { + "type": "object", + "properties": { + "linkedInDistributionTarget": { + "type": "object", + "properties": { + "visibleToGuest": { + "type": "boolean" + } + } + } + } + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json new file mode 100644 index 0000000000000..18f07c4bbb778 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json @@ -0,0 +1,8 @@ +{ + "type": "object", + "properties": { + "firstDegreeSize": { + "type": "integer" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json new file mode 100644 index 0000000000000..f95bb41047b7e --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json @@ -0,0 +1,302 @@ +{ + "type": "object", + "properties": { + "paging": { + "type": "object", + "properties": { + "start": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "links": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "rel": { + "type": "string" + }, + "href": { + "type": "string" + } + } + } + ] + }, + "total": { + "type": "integer" + } + } + }, + "elements": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "lifecycleState": { + "type": "string" + }, + "specificContent": { + "type": "object", + "properties": { + "com.linkedin.ugc.ShareContent": { + "type": "object", + "properties": { + "shareCommentary": { + "type": "object", + "properties": { + "inferredLocale": { + "type": "string" + }, + "attributes": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "length": { + "type": "integer" + }, + "start": { + "type": "integer" + }, + "value": { + "type": "object", + "properties": { + "com.linkedin.common.MemberAttributedEntity": { + "type": "object", + "properties": { + "member": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "length": { + "type": "integer" + }, + "start": { + "type": "integer" + }, + "value": { + "type": "object", + "properties": { + "com.linkedin.common.HashtagAttributedEntity": { + "type": "object", + "properties": { + "hashtag": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "length": { + "type": "integer" + }, + "start": { + "type": "integer" + }, + "value": { + "type": "object", + "properties": { + "com.linkedin.common.HashtagAttributedEntity": { + "type": "object", + "properties": { + "hashtag": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "length": { + "type": "integer" + }, + "start": { + "type": "integer" + }, + "value": { + "type": "object", + "properties": { + "com.linkedin.common.HashtagAttributedEntity": { + "type": "object", + "properties": { + "hashtag": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "length": { + "type": "integer" + }, + "start": { + "type": "integer" + }, + "value": { + "type": "object", + "properties": { + "com.linkedin.common.HashtagAttributedEntity": { + "type": "object", + "properties": { + "hashtag": { + "type": "string" + } + } + } + } + } + } + } + ] + }, + "text": { + "type": "string" + } + } + }, + "media": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "thumbnails": { + "type": "array", + "items": {} + }, + "status": { + "type": "string" + }, + "media": { + "type": "string" + } + } + } + ] + }, + "shareFeatures": { + "type": "object", + "properties": { + "hashtags": { + "type": "array", + "items": [ + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + } + ] + } + } + }, + "shareMediaCategory": { + "type": "string" + } + } + } + } + }, + "visibility": { + "type": "object", + "properties": { + "com.linkedin.ugc.MemberNetworkVisibility": { + "type": "string" + } + } + }, + "created": { + "type": "object", + "properties": { + "actor": { + "type": "string" + }, + "time": { + "type": "integer" + } + } + }, + "author": { + "type": "string" + }, + "versionTag": { + "type": "string" + }, + "id": { + "type": "string" + }, + "firstPublishedAt": { + "type": "integer" + }, + "lastModified": { + "type": "object", + "properties": { + "actor": { + "type": "string" + }, + "time": { + "type": "integer" + } + } + }, + "distribution": { + "type": "object", + "properties": { + "externalDistributionChannels": { + "type": "array", + "items": {} + }, + "distributedViaFollowFeed": { + "type": "boolean" + }, + "feedDistribution": { + "type": "string" + } + } + }, + "contentCertificationRecord": { + "type": "string" + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py new file mode 100644 index 0000000000000..214923cc0f94c --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py @@ -0,0 +1,166 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator + + +class LinkedinPagesStream(HttpStream, ABC): + + url_base = "https://api.linkedin.com/v2/" + primary_key = None + + def __init__(self, config): + super().__init__(authenticator=config.get("authenticator")) + self.config = config + + + @property + def org(self): + """Property to return the user Organization Id from input""" + return self.config.get("org_id") + + def path(self, **kwargs) -> str: + """Returns the API endpoint path for stream, from `endpoint` class attribute.""" + return self.endpoint + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None + ) -> Iterable[Mapping]: + return [response.json()] + +class OrganizationLookup(LinkedinPagesStream): + + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"organizations/{self.org}" + return path + +class FollowerStatistics(LinkedinPagesStream): + + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"organizationalEntityFollowerStatistics?q=organizationalEntity&organizationalEntity=urn:li:organization:{self.org}" + return path + +class PageStatistics(LinkedinPagesStream): + + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"organizationPageStatistics?q=organization&organization=urn%3Ali%3Aorganization%3A{self.org}" + return path + +class ShareStatistics(LinkedinPagesStream): + + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"organizationalEntityShareStatistics?q=organizationalEntity&organizationalEntity=urn%3Ali%3Aorganization%3A{self.org}" + return path + +class Shares(LinkedinPagesStream): + + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"shares?q=owners&owners=urn%3Ali%3Aorganization%3A{self.org}&sortBy=LAST_MODIFIED&sharesPerOwner=50" + return path + +class TotalFollowerCount(LinkedinPagesStream): + + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"networkSizes/urn:li:organization:{self.org}?edgeType=CompanyFollowedByMember" + return path + +class UgcPosts(LinkedinPagesStream): + + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"ugcPosts?q=authors&authors=List(urn%3Ali%3Aorganization%3A{self.org})&sortBy=LAST_MODIFIED&count=50" + return path + + def request_headers(self, stream_state: Mapping[str, Any], **kwargs) -> Mapping[str, Any]: + """ + If org_ids are specified as user's input from configuration, + we must use MODIFIED header: {'X-RestLi-Protocol-Version': '2.0.0'} + """ + return {"X-RestLi-Protocol-Version": "2.0.0"} if self.org else {} + + +class SourceLinkedinPages(AbstractSource): + """ + Abstract Source inheritance, provides: + - implementation for `check` connector's connectivity + - implementation to call each stream with it's input parameters. + """ + + @classmethod + def get_authenticator(cls, config: Mapping[str, Any]) -> TokenAuthenticator: + """ + Validate input parameters and generate a necessary Authentication object + This connectors support 2 auth methods: + 1) direct access token with TTL = 2 months + 2) refresh token (TTL = 1 year) which can be converted to access tokens + Every new refresh revokes all previous access tokens q + """ + auth_method = config.get("credentials", {}).get("auth_method") + if not auth_method or auth_method == "access_token": + # support of backward compatibility with old exists configs + access_token = config["credentials"]["access_token"] if auth_method else config["access_token"] + return TokenAuthenticator(token=access_token) + elif auth_method == "oAuth2.0": + return Oauth2Authenticator( + token_refresh_endpoint="https://www.linkedin.com/oauth/v2/accessToken", + client_id=config["credentials"]["client_id"], + client_secret=config["credentials"]["client_secret"], + refresh_token=config["credentials"]["refresh_token"], + ) + raise Exception("incorrect input parameters") + + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: + # RUN $ python main.py check --config secrets/config.json + + """ + Testing connection availability for the connector. + :: for this check method the Customer must have the "r_liteprofile" scope enabled. + :: more info: https://docs.microsoft.com/linkedin/consumer/integrations/self-serve/sign-in-with-linkedin + """ + + config["authenticator"] = self.get_authenticator(config) + stream = OrganizationLookup(config) + stream.records_limit = 1 + try: + next(stream.read_records(sync_mode=SyncMode.full_refresh), None) + return True, None + except Exception as e: + return False, e + + # RUN: $ python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + config["authenticator"] = self.get_authenticator(config) + return [ + OrganizationLookup(config), + FollowerStatistics(config), + PageStatistics(config), + ShareStatistics(config), + Shares(config), + TotalFollowerCount(config), + UgcPosts(config) + ] + diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json new file mode 100644 index 0000000000000..be64213a3de8f --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json @@ -0,0 +1,78 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/linkedin-pages/", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Linkedin Pages Spec", + "type": "object", + "required": ["org_id"], + "additionalProperties": true, + "properties": { + "org_id": { + "title": "Organization ID", + "type": "integer", + "description": "Specify the Organization ID", + "examples": [123456789] + }, + "credentials": { + "title": "Authentication *", + "type": "object", + "oneOf": [ + { + "type": "object", + "title": "OAuth2.0", + "required": ["client_id", "client_secret", "refresh_token"], + "properties": { + "auth_method": { + "type": "string", + "const": "oAuth2.0" + }, + "client_id": { + "type": "string", + "title": "Client ID", + "description": "The client ID of the LinkedIn developer application.", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "title": "Client secret", + "description": "The client secret of the LinkedIn developer application.", + "airbyte_secret": true + }, + "refresh_token": { + "type": "string", + "title": "Refresh token", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours.", + "airbyte_secret": true + } + } + }, + { + "title": "Access token", + "type": "object", + "required": ["access_token"], + "properties": { + "auth_method": { + "type": "string", + "const": "access_token" + }, + "access_token": { + "type": "string", + "title": "Access token", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours.", + "airbyte_secret": true + } + } + } + ] + } + } + }, + "authSpecification": { + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": ["credentials", "0"], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["refresh_token"]] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py new file mode 100644 index 0000000000000..745ed95943ea2 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py @@ -0,0 +1,324 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import json +from typing import Any, Dict, Iterable, List, Mapping + +import pendulum as pdm + + +def get_parent_stream_values(record: Dict, key_value_map: Dict) -> Dict: + """ + Outputs the Dict with key:value slices for the stream. + :: EXAMPLE: + Input: + records = [{dict}, {dict}, ...], + key_value_map = {: } + + Output: + { + : records..value, + } + """ + result = {} + for key in key_value_map: + value = record.get(key_value_map[key]) + if value: + result[key] = value + return result + + +def transform_change_audit_stamps( + record: Dict, dict_key: str = "changeAuditStamps", props: List = ["created", "lastModified"], fields: List = ["time"] +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `changeAuditStamps` input structure: + { + "changeAuditStamps": { + "created": {"time": 1629581275000}, + "lastModified": {"time": 1629664544760} + } + } + + :: EXAMPLE output: + { + "created": "2021-08-21 21:27:55", + "lastModified": "2021-08-22 20:35:44" + } + """ + + target_dict: Dict = record.get(dict_key) + for prop in props: + # Update dict with flatten key:value + for field in fields: + record[prop] = pdm.from_timestamp(target_dict.get(prop).get(field) / 1000).to_datetime_string() + record.pop(dict_key) + + return record + + +def date_str_from_date_range(record: Dict, prefix: str) -> str: + """ + Makes the ISO8601 format date string from the input . + + EXAMPLE: + Input: record + { + "start.year": 2021, "start.month": 8, "start.day": 1, + "end.year": 2021, "end.month": 9, "end.day": 31 + } + + EXAMPLE output: + With `prefix` = "start" + str: "2021-08-13", + + With `prefix` = "end" + str: "2021-09-31", + """ + + year = record.get(f"{prefix}.year") + month = record.get(f"{prefix}.month") + day = record.get(f"{prefix}.day") + return pdm.date(year, month, day).to_date_string() + + +def transform_date_range( + record: Dict, + dict_key: str = "dateRange", + props: List = ["start", "end"], + fields: List = ["year", "month", "day"], +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `dateRange` input structure in Analytics streams: + { + "dateRange": { + "start": {"month": 8, "day": 13, "year": 2021}, + "end": {"month": 8, "day": 13, "year": 2021} + } + } + :: EXAMPLE output: + { + "start_date": "2021-08-13", + "end_date": "2021-08-13" + } + """ + # define list of tmp keys for cleanup. + keys_to_remove = [dict_key, "start.day", "start.month", "start.year", "end.day", "end.month", "end.year", "start", "end"] + + target_dict: Dict = record.get(dict_key) + for prop in props: + # Update dict with flatten key:value + for field in fields: + record.update(**{f"{prop}.{field}": target_dict.get(prop).get(field)}) + # We build `start_date` & `end_date` fields from nested structure. + record.update(**{"start_date": date_str_from_date_range(record, "start"), "end_date": date_str_from_date_range(record, "end")}) + # Cleanup tmp fields & nested used parts + for key in keys_to_remove: + if key in record: + record.pop(key) + return record + + +def transform_targeting_criteria( + record: Dict, + dict_key: str = "targetingCriteria", +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `targetingCriteria` input structure: + { + "targetingCriteria": { + "include": { + "and": [ + { + "or": { + "urn:li:adTargetingFacet:titles": [ + "urn:li:title:100", + "urn:li:title:10326", + "urn:li:title:10457", + "urn:li:title:10738", + "urn:li:title:10966", + "urn:li:title:11349", + "urn:li:title:1159", + ] + } + }, + {"or": {"urn:li:adTargetingFacet:locations": ["urn:li:geo:103644278"]}}, + {"or": {"urn:li:adTargetingFacet:interfaceLocales": ["urn:li:locale:en_US"]}}, + ] + }, + "exclude": { + "or": { + "urn:li:adTargetingFacet:facet_Key1": [ + "facet_test1", + "facet_test2", + ], + "urn:li:adTargetingFacet:facet_Key2": [ + "facet_test3", + "facet_test4", + ], + } + } + } + + :: EXAMPLE output: + { + "targetingCriteria": { + "include": { + "and": [ + { + "type": "urn:li:adTargetingFacet:titles", + "values": [ + "urn:li:title:100", + "urn:li:title:10326", + "urn:li:title:10457", + "urn:li:title:10738", + "urn:li:title:10966", + "urn:li:title:11349", + "urn:li:title:1159", + ], + }, + { + "type": "urn:li:adTargetingFacet:locations", + "values": ["urn:li:geo:103644278"], + }, + { + "type": "urn:li:adTargetingFacet:interfaceLocales", + "values": ["urn:li:locale:en_US"], + }, + ] + }, + "exclude": { + "or": [ + { + "type": "urn:li:adTargetingFacet:facet_Key1", + "values": ["facet_test1", "facet_test2"], + }, + { + "type": "urn:li:adTargetingFacet:facet_Key2", + "values": ["facet_test3", "facet_test4"], + }, + ] + }, + } + + """ + + def unnest_dict(nested_dict: Dict) -> Iterable[Dict]: + """ + Unnest the nested dict to simplify the normalization + + EXAMPLE OUTPUT: + [ + {"type": "some_key", "values": "some_values"}, + ..., + {"type": "some_other_key", "values": "some_other_values"} + ] + """ + + for key, value in nested_dict.items(): + values = [] + if isinstance(value, List): + if len(value) > 0: + if isinstance(value[0], str): + values = value + elif isinstance(value[0], Dict): + for v in value: + values.append(v) + elif isinstance(value, Dict): + values.append(value) + yield {"type": key, "values": values} + + # get the target dict from record + targeting_criteria = record.get(dict_key) + + # transform `include` + if "include" in targeting_criteria: + and_list = targeting_criteria.get("include").get("and") + updated_include = {"and": []} + for k in and_list: + or_dict = k.get("or") + for j in unnest_dict(or_dict): + updated_include["and"].append(j) + # Replace the original 'and' with updated_include + record["targetingCriteria"]["include"] = updated_include + + # transform `exclude` if present + if "exclude" in targeting_criteria: + or_dict = targeting_criteria.get("exclude").get("or") + updated_exclude = {"or": []} + for k in unnest_dict(or_dict): + updated_exclude["or"].append(k) + # Replace the original 'or' with updated_exclude + record["targetingCriteria"]["exclude"] = updated_exclude + + return record + + +def transform_variables( + record: Dict, + dict_key: str = "variables", +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `variables` input: + { + "variables": { + "data": { + "com.linkedin.ads.SponsoredUpdateCreativeVariables": { + "activity": "urn:li:activity:1234", + "directSponsoredContent": 0, + "share": "urn:li:share:1234", + } + } + } + } + + :: EXAMPLE output: + { + "variables": { + "type": "com.linkedin.ads.SponsoredUpdateCreativeVariables", + "values": [ + {"key": "activity", "value": "urn:li:activity:1234"}, + {"key": "directSponsoredContent", "value": 0}, + {"key": "share", "value": "urn:li:share:1234"}, + ], + } + } + """ + + variables = record.get(dict_key).get("data") + for key, params in variables.items(): + record["variables"]["type"] = key + record["variables"]["values"] = [] + for key, value in params.items(): + # convert various datatypes of values into the string + record["variables"]["values"].append({"key": key, "value": json.dumps(value, ensure_ascii=True)}) + # Clean the nested structure + record["variables"].pop("data") + return record + + +def transform_data(records: List) -> Iterable[Mapping]: + """ + We need to transform the nested complex data structures into simple key:value pair, + to be properly normalised in the destination. + """ + for record in records: + + if "changeAuditStamps" in record: + record = transform_change_audit_stamps(record) + + if "dateRange" in record: + record = transform_date_range(record) + + if "targetingCriteria" in record: + record = transform_targeting_criteria(record) + + if "variables" in record: + record = transform_variables(record) + + yield record \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/__init__.py b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_incremental_streams.py new file mode 100644 index 0000000000000..4b7913646e2ac --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_incremental_streams.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_linkedin_pages.source import IncrementalLinkedinPagesStream + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalLinkedinPagesStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalLinkedinPagesStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalLinkedinPagesStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalLinkedinPagesStream() + # TODO: replace this with your expected cursor field + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class): + stream = IncrementalLinkedinPagesStream() + # TODO: replace this with your input parameters + inputs = {"current_stream_state": None, "latest_record": None} + # TODO: replace this with your expected updated stream state + expected_state = {} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalLinkedinPagesStream() + # TODO: replace this with your input parameters + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + # TODO: replace this with your expected stream slices list + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalLinkedinPagesStream, "cursor_field", "dummy_field") + stream = IncrementalLinkedinPagesStream() + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalLinkedinPagesStream() + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalLinkedinPagesStream() + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py new file mode 100644 index 0000000000000..7f23f89ed1806 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_linkedin_pages.source import SourceLinkedinPages + + +def test_check_connection(mocker): + source = SourceLinkedinPages() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceLinkedinPages() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_streams.py new file mode 100644 index 0000000000000..203fc70bb3dfa --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_streams.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_linkedin_pages.source import LinkedinPagesStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(LinkedinPagesStream, "path", "v0/example_endpoint") + mocker.patch.object(LinkedinPagesStream, "primary_key", "test_primary_key") + mocker.patch.object(LinkedinPagesStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = LinkedinPagesStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = LinkedinPagesStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected next page token + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = LinkedinPagesStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = LinkedinPagesStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = LinkedinPagesStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = LinkedinPagesStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = LinkedinPagesStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time From 3661d7d724e62f4256b37986c66263c40aec6f59 Mon Sep 17 00:00:00 2001 From: Jordan Scott Date: Mon, 23 May 2022 13:46:52 -0500 Subject: [PATCH 02/21] fixed last source acceptance test issue and passed all --- .vscode/settings.json | 7 +- CODE_OF_CONDUCT 2.md | 2 + CONTRIBUTING 2.md | 2 + airbyte-integrations/builds.md | 1 + .../source-linkedin-pages/Dockerfile | 2 +- .../acceptance-test-config.yml | 15 +- .../schemas/follower_statistics.json | 191 +- .../schemas/organization_lookup.json | 182 +- .../schemas/page_statistics.json | 1716 +---------------- .../schemas/share_statistics.json | 44 +- .../source_linkedin_pages/schemas/shares.json | 100 +- .../schemas/total_follower_count.json | 4 +- .../schemas/ugc_posts.json | 293 +-- .../source_linkedin_pages/source.py | 12 + .../unit_tests/test_source.py | 5 +- airbyte_github_hg_social_repo_url | 5 + codecov 2.yml | 18 + docs/integrations/README.md | 1 + docs/integrations/sources/linkedin-pages.md | 112 ++ 19 files changed, 257 insertions(+), 2455 deletions(-) create mode 100644 CODE_OF_CONDUCT 2.md create mode 100644 CONTRIBUTING 2.md create mode 100644 airbyte_github_hg_social_repo_url create mode 100644 codecov 2.yml create mode 100644 docs/integrations/sources/linkedin-pages.md diff --git a/.vscode/settings.json b/.vscode/settings.json index f033cb881e892..3c39f6028ac33 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -31,6 +31,9 @@ }, "[json]": { "editor.formatOnSave": true, - "editor.defaultFormatter": "esbenp.prettier-vscode" - } + "editor.defaultFormatter": "vscode.json-language-features" + }, + "python.analysis.extraPaths": [ + "./airbyte-cdk/python" + ] } diff --git a/CODE_OF_CONDUCT 2.md b/CODE_OF_CONDUCT 2.md new file mode 100644 index 0000000000000..7d0ec94e63928 --- /dev/null +++ b/CODE_OF_CONDUCT 2.md @@ -0,0 +1,2 @@ +# Code of conduct +View in [docs.airbyte.io](https://docs.airbyte.io/contributing-to-airbyte/code-of-conduct) diff --git a/CONTRIBUTING 2.md b/CONTRIBUTING 2.md new file mode 100644 index 0000000000000..85512b1d4afa6 --- /dev/null +++ b/CONTRIBUTING 2.md @@ -0,0 +1,2 @@ +# Contributing +View on [docs.airbyte.io](https://docs.airbyte.io/contributing-to-airbyte) diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 64ab8590280a5..e026ede90fcc6 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -50,6 +50,7 @@ | Iterable | [![source-iterable](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-iterable%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-iterable) | | Jira | [![source-jira](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-jira%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-jira) | | LinkedIn Ads | [![source-linkedin-ads](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-linkedin-ads%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-linkedin-ads) | +| LinkedIn Pages | | | Linnworks | [![source-linnworks](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-linnworks%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-linnworks) | | Lever Hiring | [![source-lever-hiring](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-lever-hiring%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-lever-hiring) | | Looker | [![source-looker](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-looker%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-looker) | diff --git a/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile b/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile index 6eff6390b8787..926f0c238fb0a 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile +++ b/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7.11-alpine3.14 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder diff --git a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml index 15191f1b4d4d6..cbdcf258c7f03 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml @@ -1,6 +1,6 @@ # See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests -connector_image: airbyte/source-linkedin-pages:dev0.1.1 +connector_image: airbyte/source-linkedin-pages:dev tests: spec: - spec_path: "source_linkedin_pages/spec.json" @@ -14,17 +14,6 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes - # incremental: # TODO if your connector does not implement incremental sync, remove this block - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + configured_catalog_path: "integration_tests/configured_catalog.json" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json index 1e1d0298f3d93..248f4dcca47f5 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -1,201 +1,24 @@ { - "type": "object", + "type": ["null", "object"], "properties": { "paging": { - "type": "object", + "type": ["null", "object"], "properties": { "start": { - "type": "integer" + "type": ["null", "integer"] }, "count": { - "type": "integer" + "type": ["null", "integer"] }, "links": { - "type": "array", + "type": ["null", "array"], "items": {} } } }, "elements": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "followerCountsByAssociationType": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "followerCounts": { - "type": "object", - "properties": { - "organicFollowerCount": { - "type": "integer" - }, - "paidFollowerCount": { - "type": "integer" - } - } - }, - "associationType": { - "type": "string" - } - } - } - ] - }, - "followerCountsByRegion": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "region": { - "type": "string" - }, - "followerCounts": { - "type": "object", - "properties": { - "organicFollowerCount": { - "type": "integer" - }, - "paidFollowerCount": { - "type": "integer" - } - } - } - } - } - ] - }, - "followerCountsBySeniority": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "followerCounts": { - "type": "object", - "properties": { - "organicFollowerCount": { - "type": "integer" - }, - "paidFollowerCount": { - "type": "integer" - } - } - }, - "seniority": { - "type": "string" - } - } - } - ] - }, - "followerCountsByIndustry": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "followerCounts": { - "type": "object", - "properties": { - "organicFollowerCount": { - "type": "integer" - }, - "paidFollowerCount": { - "type": "integer" - } - } - }, - "industry": { - "type": "string" - } - } - } - ] - }, - "followerCountsByFunction": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "followerCounts": { - "type": "object", - "properties": { - "organicFollowerCount": { - "type": "integer" - }, - "paidFollowerCount": { - "type": "integer" - } - } - }, - "function": { - "type": "string" - } - } - } - ] - }, - "followerCountsByStaffCountRange": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "followerCounts": { - "type": "object", - "properties": { - "organicFollowerCount": { - "type": "integer" - }, - "paidFollowerCount": { - "type": "integer" - } - } - }, - "staffCountRange": { - "type": "string" - } - } - } - ] - }, - "followerCountsByCountry": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "followerCounts": { - "type": "object", - "properties": { - "organicFollowerCount": { - "type": "integer" - }, - "paidFollowerCount": { - "type": "integer" - } - } - }, - "country": { - "type": "string" - } - } - } - ] - }, - "organizationalEntity": { - "type": "string" - } - } - } - ] + "type": ["null", "array"], + "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json index 11e1446820854..badbb4746301e 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -1,268 +1,202 @@ { - "type": "object", + "type": ["null", "object"], "properties": { "vanityName": { - "type": "string" + "type": ["null", "string"] }, "localizedName": { - "type": "string" + "type": ["null", "string"] }, "website": { - "type": "object", + "type": ["null", "object"], "properties": { "localized": { - "type": "object", + "type": ["null", "object"], "properties": { "en_US": { - "type": "string" + "type": ["null", "string"] } } }, "preferredLocale": { - "type": "object", + "type": ["null", "object"], "properties": { "country": { - "type": "string" + "type": ["null", "string"] }, "language": { - "type": "string" + "type": ["null", "string"] } } } } }, "foundedOn": { - "type": "object", + "type": ["null", "object"], "properties": { "year": { - "type": "integer" + "type": ["null", "integer"] } } }, "groups": { - "type": "array", + "type": ["null", "array"], "items": {} }, "description": { - "type": "object", + "type": ["null", "object"], "properties": { "localized": { - "type": "object", + "type": ["null", "object"], "properties": { "en_US": { - "type": "string" + "type": ["null", "string"] } } }, "preferredLocale": { - "type": "object", + "type": ["null", "object"], "properties": { "country": { - "type": "string" + "type": ["null", "string"] }, "language": { - "type": "string" + "type": ["null", "string"] } } } } }, "versionTag": { - "type": "string" + "type": ["null", "string"] }, "coverPhotoV2": { - "type": "object", + "type": ["null", "object"], "properties": { "cropped": { - "type": "string" + "type": ["null", "string"] }, "original": { - "type": "string" + "type": ["null", "string"] }, "cropInfo": { - "type": "object", + "type": ["null", "object"], "properties": { "x": { - "type": "integer" + "type": ["null", "integer"] }, "width": { - "type": "integer" + "type": ["null", "integer"] }, "y": { - "type": "integer" + "type": ["null", "integer"] }, "height": { - "type": "integer" + "type": ["null", "integer"] } } } } }, "defaultLocale": { - "type": "object", + "type": ["null", "object"], "properties": { "country": { - "type": "string" + "type": ["null", "string"] }, "language": { - "type": "string" + "type": ["null", "string"] } } }, "organizationType": { - "type": "string" + "type": ["null", "string"] }, "alternativeNames": { - "type": "array", + "type": ["null", "array"], "items": {} }, "specialties": { - "type": "array", + "type": ["null", "array"], "items": {} }, "staffCountRange": { - "type": "string" + "type": ["null", "string"] }, "localizedSpecialties": { - "type": "array", + "type": ["null", "array"], "items": {} }, "industries": { - "type": "array", - "items": [ - { - "type": "string" - } - ] + "type": ["null", "array"], + "items": {} }, "name": { - "type": "object", + "type": ["null", "object"], "properties": { "localized": { - "type": "object", + "type": ["null", "object"], "properties": { "en_US": { - "type": "string" + "type": ["null", "string"] } } }, "preferredLocale": { - "type": "object", + "type": ["null", "object"], "properties": { "country": { - "type": "string" + "type": ["null", "string"] }, "language": { - "type": "string" + "type": ["null", "string"] } } } } }, "primaryOrganizationType": { - "type": "string" + "type": ["null", "string"] }, "locations": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "locationType": { - "type": "string" - }, - "description": { - "type": "object", - "properties": { - "localized": { - "type": "object", - "properties": { - "en_US": { - "type": "string" - } - } - }, - "preferredLocale": { - "type": "object", - "properties": { - "country": { - "type": "string" - }, - "language": { - "type": "string" - } - } - } - } - }, - "address": { - "type": "object", - "properties": { - "geographicArea": { - "type": "string" - }, - "country": { - "type": "string" - }, - "city": { - "type": "string" - }, - "line1": { - "type": "string" - }, - "postalCode": { - "type": "string" - } - } - }, - "localizedDescription": { - "type": "string" - }, - "geoLocation": { - "type": "string" - }, - "streetAddressFieldState": { - "type": "string" - } - } - } - ] + "type": ["null", "array"], + "items": {} }, "id": { - "type": "integer" + "type": ["null", "integer"] }, "localizedDescription": { - "type": "string" + "type": ["null", "string"] }, "$URN": { - "type": "string" + "type": ["null", "string"] }, "localizedWebsite": { - "type": "string" + "type": ["null", "string"] }, "logoV2": { - "type": "object", + "type": ["null", "object"], "properties": { "cropped": { - "type": "string" + "type": ["null", "string"] }, "original": { - "type": "string" + "type": ["null", "string"] }, "cropInfo": { - "type": "object", + "type": ["null", "object"], "properties": { "x": { - "type": "integer" + "type": ["null", "integer"] }, "width": { - "type": "integer" + "type": ["null", "integer"] }, "y": { - "type": "integer" + "type": ["null", "integer"] }, "height": { - "type": "integer" + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json index a003e0efc4bc0..248f4dcca47f5 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json @@ -1,1726 +1,24 @@ { - "type": "object", + "type": ["null", "object"], "properties": { "paging": { - "type": "object", + "type": ["null", "object"], "properties": { "start": { - "type": "integer" + "type": ["null", "integer"] }, "count": { - "type": "integer" + "type": ["null", "integer"] }, "links": { - "type": "array", + "type": ["null", "array"], "items": {} } } }, "elements": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "pageStatisticsBySeniority": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "pageStatistics": { - "type": "object", - "properties": { - "views": { - "type": "object", - "properties": { - "mobileProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allDesktopPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "insightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allMobilePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "jobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "productsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "peoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "overviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "lifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "careersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopPeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "aboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobilePeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - } - } - } - } - }, - "seniority": { - "type": "string" - } - } - } - ] - }, - "pageStatisticsByCountry": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "country": { - "type": "string" - }, - "pageStatistics": { - "type": "object", - "properties": { - "views": { - "type": "object", - "properties": { - "mobileProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allDesktopPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "insightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allMobilePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "jobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "productsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "peoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "overviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "lifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "careersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopPeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "aboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobilePeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - } - } - } - } - } - } - } - ] - }, - "pageStatisticsByIndustry": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "pageStatistics": { - "type": "object", - "properties": { - "views": { - "type": "object", - "properties": { - "mobileProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allDesktopPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "insightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allMobilePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "jobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "productsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "peoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "overviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "lifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "careersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopPeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "aboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobilePeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - } - } - } - } - } - } - } - ] - }, - "organization": { - "type": "string" - }, - "totalPageStatistics": { - "type": "object", - "properties": { - "clicks": { - "type": "object", - "properties": { - "mobileCareersPageClicks": { - "type": "object", - "properties": { - "careersPageJobsClicks": { - "type": "integer" - }, - "careersPagePromoLinksClicks": { - "type": "integer" - }, - "careersPageEmployeesClicks": { - "type": "integer" - } - } - }, - "careersPageClicks": { - "type": "object", - "properties": { - "careersPagePromoLinksClicks": { - "type": "integer" - }, - "careersPageBannerPromoClicks": { - "type": "integer" - }, - "careersPageJobsClicks": { - "type": "integer" - }, - "careersPageEmployeesClicks": { - "type": "integer" - } - } - } - } - }, - "views": { - "type": "object", - "properties": { - "mobileProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allDesktopPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "insightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allMobilePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "jobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "productsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "peoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "overviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "lifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "careersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopPeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "aboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobilePeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - } - } - } - } - }, - "pageStatisticsByStaffCountRange": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "staffCountRange": { - "type": "string" - }, - "pageStatistics": { - "type": "object", - "properties": { - "views": { - "type": "object", - "properties": { - "mobileProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allDesktopPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "insightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allMobilePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "jobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "productsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "peoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "overviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "lifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "careersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopPeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "aboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobilePeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - } - } - } - } - } - } - } - ] - }, - "pageStatisticsByRegion": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "region": { - "type": "string" - }, - "pageStatistics": { - "type": "object", - "properties": { - "views": { - "type": "object", - "properties": { - "mobileProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allDesktopPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "insightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allMobilePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "jobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "productsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "peoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "overviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "lifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "careersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopPeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "aboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobilePeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - } - } - } - } - } - } - } - ] - }, - "pageStatisticsByFunction": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "pageStatistics": { - "type": "object", - "properties": { - "views": { - "type": "object", - "properties": { - "mobileProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allDesktopPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "insightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allMobilePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "jobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "productsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "peoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "overviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "lifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "careersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopPeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "aboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobilePeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - } - } - } - } - }, - "function": { - "type": "string" - } - } - } - ] - } - } - } - ] + "type": ["null", "array"], + "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json index dfc9a7e6a7115..2bea295e87bf9 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json @@ -1,4 +1,5 @@ { + "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { "paging": { @@ -21,48 +22,7 @@ }, "elements": { "type": "array", - "items": [ - { - "type": "object", - "properties": { - "totalShareStatistics": { - "type": "object", - "properties": { - "uniqueImpressionsCount": { - "type": "integer" - }, - "shareCount": { - "type": "integer" - }, - "shareMentionsCount": { - "type": "integer" - }, - "engagement": { - "type": "number" - }, - "clickCount": { - "type": "integer" - }, - "likeCount": { - "type": "integer" - }, - "impressionCount": { - "type": "integer" - }, - "commentMentionsCount": { - "type": "integer" - }, - "commentCount": { - "type": "integer" - } - } - }, - "organizationalEntity": { - "type": "string" - } - } - } - ] + "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json index 5e0fd75a811ce..621997bd13b52 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json @@ -1,109 +1,27 @@ { - "type": "object", + "type": ["null", "object"], "properties": { "paging": { - "type": "object", + "type": ["null", "object"], "properties": { "start": { - "type": "integer" + "type": ["null", "integer"] }, "count": { - "type": "integer" + "type": ["null", "integer"] }, "links": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "rel": { - "type": "string" - }, - "href": { - "type": "string" - } - } - } - ] + "type": ["null", "array"], + "items": {} }, "total": { - "type": "integer" + "type": ["null", "integer"] } } }, "elements": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "owner": { - "type": "string" - }, - "activity": { - "type": "string" - }, - "edited": { - "type": "boolean" - }, - "resharedShare": { - "type": "string" - }, - "created": { - "type": "object", - "properties": { - "actor": { - "type": "string" - }, - "time": { - "type": "integer" - } - } - }, - "lastModified": { - "type": "object", - "properties": { - "actor": { - "type": "string" - }, - "time": { - "type": "integer" - } - } - }, - "text": { - "type": "object", - "properties": { - "text": { - "type": "string" - } - } - }, - "id": { - "type": "string" - }, - "originalShare": { - "type": "string" - }, - "distribution": { - "type": "object", - "properties": { - "linkedInDistributionTarget": { - "type": "object", - "properties": { - "visibleToGuest": { - "type": "boolean" - } - } - } - } - } - } - } - ] + "type": ["null", "array"], + "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json index 18f07c4bbb778..d48ce8aa10a17 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json @@ -1,8 +1,8 @@ { - "type": "object", + "type": ["null", "object"], "properties": { "firstDegreeSize": { - "type": "integer" + "type": ["null", "integer"] } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json index f95bb41047b7e..621997bd13b52 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json @@ -1,302 +1,27 @@ { - "type": "object", + "type": ["null", "object"], "properties": { "paging": { - "type": "object", + "type": ["null", "object"], "properties": { "start": { - "type": "integer" + "type": ["null", "integer"] }, "count": { - "type": "integer" + "type": ["null", "integer"] }, "links": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "rel": { - "type": "string" - }, - "href": { - "type": "string" - } - } - } - ] + "type": ["null", "array"], + "items": {} }, "total": { - "type": "integer" + "type": ["null", "integer"] } } }, "elements": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "lifecycleState": { - "type": "string" - }, - "specificContent": { - "type": "object", - "properties": { - "com.linkedin.ugc.ShareContent": { - "type": "object", - "properties": { - "shareCommentary": { - "type": "object", - "properties": { - "inferredLocale": { - "type": "string" - }, - "attributes": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "length": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "value": { - "type": "object", - "properties": { - "com.linkedin.common.MemberAttributedEntity": { - "type": "object", - "properties": { - "member": { - "type": "string" - } - } - } - } - } - } - }, - { - "type": "object", - "properties": { - "length": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "value": { - "type": "object", - "properties": { - "com.linkedin.common.HashtagAttributedEntity": { - "type": "object", - "properties": { - "hashtag": { - "type": "string" - } - } - } - } - } - } - }, - { - "type": "object", - "properties": { - "length": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "value": { - "type": "object", - "properties": { - "com.linkedin.common.HashtagAttributedEntity": { - "type": "object", - "properties": { - "hashtag": { - "type": "string" - } - } - } - } - } - } - }, - { - "type": "object", - "properties": { - "length": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "value": { - "type": "object", - "properties": { - "com.linkedin.common.HashtagAttributedEntity": { - "type": "object", - "properties": { - "hashtag": { - "type": "string" - } - } - } - } - } - } - }, - { - "type": "object", - "properties": { - "length": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "value": { - "type": "object", - "properties": { - "com.linkedin.common.HashtagAttributedEntity": { - "type": "object", - "properties": { - "hashtag": { - "type": "string" - } - } - } - } - } - } - } - ] - }, - "text": { - "type": "string" - } - } - }, - "media": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "thumbnails": { - "type": "array", - "items": {} - }, - "status": { - "type": "string" - }, - "media": { - "type": "string" - } - } - } - ] - }, - "shareFeatures": { - "type": "object", - "properties": { - "hashtags": { - "type": "array", - "items": [ - { - "type": "string" - }, - { - "type": "string" - }, - { - "type": "string" - }, - { - "type": "string" - } - ] - } - } - }, - "shareMediaCategory": { - "type": "string" - } - } - } - } - }, - "visibility": { - "type": "object", - "properties": { - "com.linkedin.ugc.MemberNetworkVisibility": { - "type": "string" - } - } - }, - "created": { - "type": "object", - "properties": { - "actor": { - "type": "string" - }, - "time": { - "type": "integer" - } - } - }, - "author": { - "type": "string" - }, - "versionTag": { - "type": "string" - }, - "id": { - "type": "string" - }, - "firstPublishedAt": { - "type": "integer" - }, - "lastModified": { - "type": "object", - "properties": { - "actor": { - "type": "string" - }, - "time": { - "type": "integer" - } - } - }, - "distribution": { - "type": "object", - "properties": { - "externalDistributionChannels": { - "type": "array", - "items": {} - }, - "distributedViaFollowFeed": { - "type": "boolean" - }, - "feedDistribution": { - "type": "string" - } - } - }, - "contentCertificationRecord": { - "type": "string" - } - } - } - ] + "type": ["null", "array"], + "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py index 214923cc0f94c..2dfa602c784c7 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py @@ -45,6 +45,18 @@ def parse_response( ) -> Iterable[Mapping]: return [response.json()] + def should_retry(self, response: requests.Response) -> bool: + if response.status_code == 429: + error_message = ( + f"Stream {self.name}: LinkedIn API requests are rate limited. " + f"Rate limits specify the maximum number of API calls that can be made in a 24 hour period. " + f"These limits reset at midnight UTC every day. " + f"You can find more information here https://docs.airbyte.io/integrations/sources/linkedin-ads. " + f"Also quotas and usage are here: https://www.linkedin.com/developers/apps." + ) + self.logger.error(error_message) + return super().should_retry(response) + class OrganizationLookup(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: diff --git a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py index 7f23f89ed1806..5609fea61d95a 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py @@ -17,6 +17,5 @@ def test_streams(mocker): source = SourceLinkedinPages() config_mock = MagicMock() streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 2 - assert len(streams) == expected_streams_number + expected_streams_number = 7 + assert len(streams) == expected_streams_number \ No newline at end of file diff --git a/airbyte_github_hg_social_repo_url b/airbyte_github_hg_social_repo_url new file mode 100644 index 0000000000000..6fce7b201a911 --- /dev/null +++ b/airbyte_github_hg_social_repo_url @@ -0,0 +1,5 @@ +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 + +https://airbyteuser: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@github.com/Hunt-Gather-Create/hg_social_media_analytics.git + +https://github.com/Hunt-Gather-Create/hg_social_media_analytics.git \ No newline at end of file diff --git a/codecov 2.yml b/codecov 2.yml new file mode 100644 index 0000000000000..55a0f1f89c3c0 --- /dev/null +++ b/codecov 2.yml @@ -0,0 +1,18 @@ +codecov: + notify: + require_ci_to_pass: no + +coverage: + status: + patch: + default: + target: 90% + if_no_uploads: error + if_not_found: failure + if_ci_failed: failure + project: + default: + target: 90% + if_no_uploads: error + if_not_found: failure + if_ci_failed: failure diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 8c07b13e2f9b5..1f8168e80f107 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -82,6 +82,7 @@ For more information about the grading system, see [Product Release Stages](http | [Lemlist](sources/lemlist.md) | Alpha | Yes | | [Lever](sources/lever-hiring.md) | Alpha | No | | [LinkedIn Ads](sources/linkedin-ads.md) | Beta | Yes | +| [LinkedIn Pages](sources/linkedin-pages.md) | Alpha | Yes | | [Linnworks](sources/linnworks.md) | Alpha | Yes | | [Looker](sources/looker.md) | Alpha | Yes | | [Magento](sources/magento.md) | Alpha | No | diff --git a/docs/integrations/sources/linkedin-pages.md b/docs/integrations/sources/linkedin-pages.md new file mode 100644 index 0000000000000..743604a7fa14b --- /dev/null +++ b/docs/integrations/sources/linkedin-pages.md @@ -0,0 +1,112 @@ +# LinkedIn Pages + +## Sync overview + +The LinkedIn Pages source only supports Full Refresh for now. Incremental Sync will be coming soon. + +This Source Connector is based on a [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). Airbyte uses [LinkedIn Marketing Developer Platform - API](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/marketing-integrations-overview) to fetch data from LinkedIn Pages. + +### Output schema + +This Source is capable of syncing the following data as streams: + +* [Organization Lookup](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/organization-lookup-api?tabs=http#retrieve-organizations) +* [Follower Statistics](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/follower-statistics?tabs=http#retrieve-lifetime-follower-statistics) +* [Page Statistics](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/page-statistics?tabs=http#retrieve-lifetime-organization-page-statistics) +* [Share Statistics](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/share-statistics?tabs=http#retrieve-lifetime-share-statistics) +* [Shares (Latest 50)](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/shares/share-api?tabs=http#find-shares-by-owner) +* [Total Follower Count](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/organization-lookup-api?tabs=http#retrieve-organization-follower-count) +* [UGC Posts](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/shares/ugc-post-api?tabs=http#find-ugc-posts-by-authors) + +### NOTE: + +All streams only sync all-time statistics at this time. A `start_date` field will be added soon to pull data starting at a single point in time. + +### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--------------- | :----------- | :------------------------- | +| `number` | `number` | float number | +| `integer` | `integer` | whole number | +| `array` | `array` | | +| `boolean` | `boolean` | True/False | +| `string` | `string` | | + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------------------------------- | :------------------- | :---- | +| Full Refresh Overwrite Sync | Yes | | +| Full Refresh Append Sync | No | | +| Incremental - Append Sync | No | | +| Incremental - Append + Deduplication Sync | No | | +| Namespaces | No | | + +### Performance considerations + +There are official Rate Limits for LinkedIn Pages API Usage, [more information here](https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/rate-limits?context=linkedin/marketing/context). Rate limited requests will receive a 429 response. Rate limits specify the maximum number of API calls that can be made in a 24 hour period. These limits reset at midnight UTC every day. In rare cases, LinkedIn may also return a 429 response as part of infrastructure protection. API service will return to normal automatically. In such cases you will receive the next error message: + +```text +"Caught retryable error ' or null' after tries. Waiting seconds then retrying..." +``` + +This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Error. If the maximum of available API requests capacity is reached, you will have the following message: + +```text +"Max try rate limit exceded..." +``` + +After 5 unsuccessful attempts - the connector will stop the sync operation. In such cases check your Rate Limits [on this page](https://www.linkedin.com/developers/apps) > Choose your app > Analytics. + +## Getting started +The API user account should be assigned the following permissions for the API endpoints: +Endpoints such as: `Organization Lookup API`, `Follower Statistics`, `Page Statistics`, `Share Statistics`, `Shares`, `UGC Posts` require these permissions: +* `r_organization_social`: Retrieve your organization's posts, comments, reactions, and other engagement data. +* `rw_organization_admin`: Manage your organization's pages and retrieve reporting data. + +The API user account should be assigned the `ADMIN` role. + +### Authentication +There are 2 authentication methods: Access Token or OAuth2.0. +OAuth2.0 is recommended since it will continue streaming data for 12 months instead of 2 months with an access token. + +##### Create the `Refresh_Token` or `Access_Token`: +The source LinkedIn Pages can use either the `client_id`, `client_secret` and `refresh_token` for OAuth2.0 authentication or simply use an `access_token` in the UI connector's settings to make API requests. Access tokens expire after `2 months from creation date (60 days)` and require a user to manually authenticate again. Refresh tokens expire after `12 months from creation date (365 days)`. If you receive a `401 invalid token response`, the error logs will state that your token has expired and to re-authenticate your connection to generate a new token. This is described more [here](https://docs.microsoft.com/en-us/linkedin/shared/authentication/authorization-code-flow?context=linkedin/context). + +1. **Log in to LinkedIn as the API user** + +2. **Create an App** [here](https://www.linkedin.com/developers/apps): + * `App Name`: airbyte-source + * `Company`: search and find your LinkedIn Company Page + * `Privacy policy URL`: link to company privacy policy + * `Business email`: developer/admin email address + * `App logo`: Airbyte's \(or Company's\) logo + * Review/agree to legal terms and create app + * Review the **Auth** tab: + * **Save your `client_id` and `client_secret`** \(for later steps\) + * Oauth 2.0 settings: Provide a `redirect_uri` \(for later steps\): `https://airbyte.io` + +3. **Verify App**: + * In the **Settings** tab of your app dashboard, you'll see a **Verify** button. Click that button! + * Generate and provide the verify URL to your Company's LinkedIn Admin to verify the app. + +4. **Request API Access**: + * Navigate to the **Products** tab + * Select the [Marketing Developer Platform](https://docs.microsoft.com/en-us/linkedin/marketing/) and agree to the legal terms + * After a few minutes, refresh the page to see a link to `View access form` in place of the **Select** button + * Fill out the access form and access should be granted **within 72 hours** (usually quicker) + +5. **Create A Refresh Token** (or Access Token): + * Navigate to the LinkedIn Developers' [OAuth Token Tools](https://www.linkedin.com/developers/tools/oauth) and click **Create token** + * Select your newly created app and check the boxes for the following scopes: + * `r_organization_social` + * `rw_organization_admin` + * Click **Request access token** and once generated, **save your Refresh token** + +6. **Use the `client_id`, `client_secret` and `refresh_token`** from Steps 2 and 5 to autorize the LinkedIn Pages connector within the Airbyte UI. + * As mentioned earlier, you can also simply use the Access token auth method for 60-day access. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------- | From df6dc8b0c1a01e4dc13e7887c08963a344ebac3a Mon Sep 17 00:00:00 2001 From: Jordan Scott Date: Tue, 31 May 2022 16:08:54 -0500 Subject: [PATCH 03/21] requested changes completed --- .vscode/settings.json | 9 +- CODE_OF_CONDUCT 2.md | 2 - CONTRIBUTING 2.md | 2 - .../acceptance-test-config.yml | 2 +- .../integration_tests/abnormal_state.json | 2 +- .../integration_tests/sample_config.json | 2 +- .../schemas/follower_statistics.json | 87 +++- .../schemas/organization_lookup.json | 416 +++++++++++++++--- .../source_linkedin_pages/source.py | 25 +- codecov 2.yml | 18 - 10 files changed, 442 insertions(+), 123 deletions(-) delete mode 100644 CODE_OF_CONDUCT 2.md delete mode 100644 CONTRIBUTING 2.md delete mode 100644 codecov 2.yml diff --git a/.vscode/settings.json b/.vscode/settings.json index 3c39f6028ac33..ebb6af501317b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -31,9 +31,6 @@ }, "[json]": { "editor.formatOnSave": true, - "editor.defaultFormatter": "vscode.json-language-features" - }, - "python.analysis.extraPaths": [ - "./airbyte-cdk/python" - ] -} + "editor.defaultFormatter": "esbenp.prettier-vscode" + } +} \ No newline at end of file diff --git a/CODE_OF_CONDUCT 2.md b/CODE_OF_CONDUCT 2.md deleted file mode 100644 index 7d0ec94e63928..0000000000000 --- a/CODE_OF_CONDUCT 2.md +++ /dev/null @@ -1,2 +0,0 @@ -# Code of conduct -View in [docs.airbyte.io](https://docs.airbyte.io/contributing-to-airbyte/code-of-conduct) diff --git a/CONTRIBUTING 2.md b/CONTRIBUTING 2.md deleted file mode 100644 index 85512b1d4afa6..0000000000000 --- a/CONTRIBUTING 2.md +++ /dev/null @@ -1,2 +0,0 @@ -# Contributing -View on [docs.airbyte.io](https://docs.airbyte.io/contributing-to-airbyte) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml index cbdcf258c7f03..b7023bc46b224 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml @@ -16,4 +16,4 @@ tests: configured_catalog_path: "integration_tests/configured_catalog.json" full_refresh: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" \ No newline at end of file + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json index ceae7331d2d9a..8f9f142bd9120 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json @@ -20,4 +20,4 @@ "ugc_posts": { "end_date": "2050-01-01" } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json index c06b53f2c96e3..fd416acb2613e 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json @@ -2,6 +2,6 @@ "org_id": 12345678, "credentials": { "auth_method": "access_token", - "access_token": "drAL4JXBJ6v1qaU9iWargosra6ibiw0ZWEdMnC0ZizeD1gLRQP6u1pkQ" + "access_token": "example_token_string123" } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json index 248f4dcca47f5..8d06eb23cb2ce 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -1,24 +1,77 @@ { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "paging": { - "type": ["null", "object"], - "properties": { - "start": { - "type": ["null", "integer"] - }, - "count": { - "type": ["null", "integer"] - }, - "links": { - "type": ["null", "array"], - "items": {} + "elements": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "followerCountsByAssociationType": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByRegion": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsBySeniority": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByIndustry": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByFunction": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByStaffCountRange": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByCountry": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "organizationalEntity": { + "type": [ + "null", + "string" + ] + } } } - }, - "elements": { - "type": ["null", "array"], - "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json index badbb4746301e..82ce72ca251f4 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -1,202 +1,494 @@ { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "vanityName": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "localizedName": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "website": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "language": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } } }, "foundedOn": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "year": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] } } }, "groups": { - "type": ["null", "array"], - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "description": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "language": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } } }, "versionTag": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "coverPhotoV2": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "cropped": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "original": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "cropInfo": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "x": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "width": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "y": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "height": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] } } } } }, "defaultLocale": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "language": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "organizationType": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "alternativeNames": { - "type": ["null", "array"], - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "specialties": { - "type": ["null", "array"], - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "staffCountRange": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "localizedSpecialties": { - "type": ["null", "array"], - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "industries": { - "type": ["null", "array"], - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } }, "name": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "language": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } } }, "primaryOrganizationType": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "locations": { - "type": ["null", "array"], - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "locationType": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "object" + ], + "properties": { + "localized": { + "type": [ + "null", + "object" + ], + "properties": { + "en_US": { + "type": [ + "null", + "string" + ] + } + } + }, + "preferredLocale": { + "type": [ + "null", + "object" + ], + "properties": { + "country": { + "type": [ + "null", + "string" + ] + }, + "language": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "address": { + "type": [ + "null", + "object" + ], + "properties": { + "geographicArea": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "line1": { + "type": [ + "null", + "string" + ] + }, + "postalCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "localizedDescription": { + "type": [ + "null", + "string" + ] + }, + "geoLocation": { + "type": [ + "null", + "string" + ] + }, + "streetAddressFieldState": { + "type": [ + "null", + "string" + ] + } + } + } }, "id": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "localizedDescription": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "$URN": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "localizedWebsite": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "logoV2": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "cropped": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "original": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "cropInfo": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "x": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "width": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "y": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "height": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] } } } diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py index 2dfa602c784c7..a8c80c5efc82c 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py @@ -23,13 +23,13 @@ class LinkedinPagesStream(HttpStream, ABC): def __init__(self, config): super().__init__(authenticator=config.get("authenticator")) self.config = config - + @property def org(self): """Property to return the user Organization Id from input""" return self.config.get("org_id") - + def path(self, **kwargs) -> str: """Returns the API endpoint path for stream, from `endpoint` class attribute.""" return self.endpoint @@ -51,7 +51,7 @@ def should_retry(self, response: requests.Response) -> bool: f"Stream {self.name}: LinkedIn API requests are rate limited. " f"Rate limits specify the maximum number of API calls that can be made in a 24 hour period. " f"These limits reset at midnight UTC every day. " - f"You can find more information here https://docs.airbyte.io/integrations/sources/linkedin-ads. " + f"You can find more information here https://docs.airbyte.io/integrations/sources/linkedin-pages. " f"Also quotas and usage are here: https://www.linkedin.com/developers/apps." ) self.logger.error(error_message) @@ -60,49 +60,49 @@ def should_retry(self, response: requests.Response) -> bool: class OrganizationLookup(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + path = f"organizations/{self.org}" return path class FollowerStatistics(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + path = f"organizationalEntityFollowerStatistics?q=organizationalEntity&organizationalEntity=urn:li:organization:{self.org}" return path class PageStatistics(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + path = f"organizationPageStatistics?q=organization&organization=urn%3Ali%3Aorganization%3A{self.org}" return path class ShareStatistics(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + path = f"organizationalEntityShareStatistics?q=organizationalEntity&organizationalEntity=urn%3Ali%3Aorganization%3A{self.org}" return path class Shares(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + path = f"shares?q=owners&owners=urn%3Ali%3Aorganization%3A{self.org}&sortBy=LAST_MODIFIED&sharesPerOwner=50" return path class TotalFollowerCount(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + path = f"networkSizes/urn:li:organization:{self.org}?edgeType=CompanyFollowedByMember" return path - + class UgcPosts(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + path = f"ugcPosts?q=authors&authors=List(urn%3Ali%3Aorganization%3A{self.org})&sortBy=LAST_MODIFIED&count=50" return path @@ -161,7 +161,7 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> return True, None except Exception as e: return False, e - + # RUN: $ python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json def streams(self, config: Mapping[str, Any]) -> List[Stream]: @@ -175,4 +175,3 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: TotalFollowerCount(config), UgcPosts(config) ] - diff --git a/codecov 2.yml b/codecov 2.yml deleted file mode 100644 index 55a0f1f89c3c0..0000000000000 --- a/codecov 2.yml +++ /dev/null @@ -1,18 +0,0 @@ -codecov: - notify: - require_ci_to_pass: no - -coverage: - status: - patch: - default: - target: 90% - if_no_uploads: error - if_not_found: failure - if_ci_failed: failure - project: - default: - target: 90% - if_no_uploads: error - if_not_found: failure - if_ci_failed: failure From 7636c4521b30b32610a61dfe5c042fdf0e8f46ed Mon Sep 17 00:00:00 2001 From: Jordan Scott Date: Mon, 23 May 2022 13:46:52 -0500 Subject: [PATCH 04/21] fixed last source acceptance test issue and passed all --- .vscode/settings.json | 2 +- .../schemas/follower_statistics.json | 87 +--- .../schemas/organization_lookup.json | 416 +++--------------- 3 files changed, 80 insertions(+), 425 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index ebb6af501317b..f033cb881e892 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -33,4 +33,4 @@ "editor.formatOnSave": true, "editor.defaultFormatter": "esbenp.prettier-vscode" } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json index 8d06eb23cb2ce..248f4dcca47f5 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -1,77 +1,24 @@ { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { - "elements": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "object" - ], - "properties": { - "followerCountsByAssociationType": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByRegion": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsBySeniority": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByIndustry": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByFunction": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByStaffCountRange": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByCountry": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "organizationalEntity": { - "type": [ - "null", - "string" - ] - } + "paging": { + "type": ["null", "object"], + "properties": { + "start": { + "type": ["null", "integer"] + }, + "count": { + "type": ["null", "integer"] + }, + "links": { + "type": ["null", "array"], + "items": {} } } + }, + "elements": { + "type": ["null", "array"], + "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json index 82ce72ca251f4..ec853967758b9 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -1,494 +1,202 @@ { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "vanityName": { - "type": [ - "null", - "string" - ] + "type": "string" }, "localizedName": { - "type": [ - "null", - "string" - ] + "type": "string" }, "website": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "localized": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "en_US": { - "type": [ - "null", - "string" - ] + "type": "string" } } }, "preferredLocale": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "country": { - "type": [ - "null", - "string" - ] + "type": "string" }, "language": { - "type": [ - "null", - "string" - ] + "type": "string" } } } } }, "foundedOn": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "year": { - "type": [ - "null", - "integer" - ] + "type": "integer" } } }, "groups": { - "type": [ - "null", - "array" - ], - "items": { - "items": {} - } + "type": "array", + "items": {} }, "description": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "localized": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "en_US": { - "type": [ - "null", - "string" - ] + "type": "string" } } }, "preferredLocale": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "country": { - "type": [ - "null", - "string" - ] + "type": "string" }, "language": { - "type": [ - "null", - "string" - ] + "type": "string" } } } } }, "versionTag": { - "type": [ - "null", - "string" - ] + "type": "string" }, "coverPhotoV2": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "cropped": { - "type": [ - "null", - "string" - ] + "type": "string" }, "original": { - "type": [ - "null", - "string" - ] + "type": "string" }, "cropInfo": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "x": { - "type": [ - "null", - "integer" - ] + "type": "integer" }, "width": { - "type": [ - "null", - "integer" - ] + "type": "integer" }, "y": { - "type": [ - "null", - "integer" - ] + "type": "integer" }, "height": { - "type": [ - "null", - "integer" - ] + "type": "integer" } } } } }, "defaultLocale": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "country": { - "type": [ - "null", - "string" - ] + "type": "string" }, "language": { - "type": [ - "null", - "string" - ] + "type": "string" } } }, "organizationType": { - "type": [ - "null", - "string" - ] + "type": "string" }, "alternativeNames": { - "type": [ - "null", - "array" - ], - "items": { - "items": {} - } + "type": "array", + "items": {} }, "specialties": { - "type": [ - "null", - "array" - ], - "items": { - "items": {} - } + "type": "array", + "items": {} }, "staffCountRange": { - "type": [ - "null", - "string" - ] + "type": "string" }, "localizedSpecialties": { - "type": [ - "null", - "array" - ], - "items": { - "items": {} - } + "type": "array", + "items": {} }, "industries": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "string" - ] - } + "type": "array", + "items": {} }, "name": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "localized": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "en_US": { - "type": [ - "null", - "string" - ] + "type": "string" } } }, "preferredLocale": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "country": { - "type": [ - "null", - "string" - ] + "type": "string" }, "language": { - "type": [ - "null", - "string" - ] + "type": "string" } } } } }, "primaryOrganizationType": { - "type": [ - "null", - "string" - ] + "type": "string" }, "locations": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "object" - ], - "properties": { - "locationType": { - "type": [ - "null", - "string" - ] - }, - "description": { - "type": [ - "null", - "object" - ], - "properties": { - "localized": { - "type": [ - "null", - "object" - ], - "properties": { - "en_US": { - "type": [ - "null", - "string" - ] - } - } - }, - "preferredLocale": { - "type": [ - "null", - "object" - ], - "properties": { - "country": { - "type": [ - "null", - "string" - ] - }, - "language": { - "type": [ - "null", - "string" - ] - } - } - } - } - }, - "address": { - "type": [ - "null", - "object" - ], - "properties": { - "geographicArea": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "city": { - "type": [ - "null", - "string" - ] - }, - "line1": { - "type": [ - "null", - "string" - ] - }, - "postalCode": { - "type": [ - "null", - "string" - ] - } - } - }, - "localizedDescription": { - "type": [ - "null", - "string" - ] - }, - "geoLocation": { - "type": [ - "null", - "string" - ] - }, - "streetAddressFieldState": { - "type": [ - "null", - "string" - ] - } - } - } + "type": "array", + "items": {} }, "id": { - "type": [ - "null", - "integer" - ] + "type": "integer" }, "localizedDescription": { - "type": [ - "null", - "string" - ] + "type": "string" }, "$URN": { - "type": [ - "null", - "string" - ] + "type": "string" }, "localizedWebsite": { - "type": [ - "null", - "string" - ] + "type": "string" }, "logoV2": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "cropped": { - "type": [ - "null", - "string" - ] + "type": "string" }, "original": { - "type": [ - "null", - "string" - ] + "type": "string" }, "cropInfo": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "x": { - "type": [ - "null", - "integer" - ] + "type": "integer" }, "width": { - "type": [ - "null", - "integer" - ] + "type": "integer" }, "y": { - "type": [ - "null", - "integer" - ] + "type": "integer" }, "height": { - "type": [ - "null", - "integer" - ] + "type": "integer" } } } From c8c1d74505fae5cec6ae53056bceb9a0a8f8793c Mon Sep 17 00:00:00 2001 From: Jordan Scott Date: Tue, 31 May 2022 16:08:54 -0500 Subject: [PATCH 05/21] requested changes completed --- .../schemas/follower_statistics.json | 87 +++- .../schemas/organization_lookup.json | 416 +++++++++++++++--- 2 files changed, 424 insertions(+), 79 deletions(-) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json index 248f4dcca47f5..8d06eb23cb2ce 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -1,24 +1,77 @@ { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "paging": { - "type": ["null", "object"], - "properties": { - "start": { - "type": ["null", "integer"] - }, - "count": { - "type": ["null", "integer"] - }, - "links": { - "type": ["null", "array"], - "items": {} + "elements": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "followerCountsByAssociationType": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByRegion": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsBySeniority": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByIndustry": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByFunction": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByStaffCountRange": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByCountry": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "organizationalEntity": { + "type": [ + "null", + "string" + ] + } } } - }, - "elements": { - "type": ["null", "array"], - "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json index ec853967758b9..82ce72ca251f4 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -1,202 +1,494 @@ { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "vanityName": { - "type": "string" + "type": [ + "null", + "string" + ] }, "localizedName": { - "type": "string" + "type": [ + "null", + "string" + ] }, "website": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": "string" + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": "string" + "type": [ + "null", + "string" + ] }, "language": { - "type": "string" + "type": [ + "null", + "string" + ] } } } } }, "foundedOn": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "year": { - "type": "integer" + "type": [ + "null", + "integer" + ] } } }, "groups": { - "type": "array", - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "description": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": "string" + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": "string" + "type": [ + "null", + "string" + ] }, "language": { - "type": "string" + "type": [ + "null", + "string" + ] } } } } }, "versionTag": { - "type": "string" + "type": [ + "null", + "string" + ] }, "coverPhotoV2": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "cropped": { - "type": "string" + "type": [ + "null", + "string" + ] }, "original": { - "type": "string" + "type": [ + "null", + "string" + ] }, "cropInfo": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "x": { - "type": "integer" + "type": [ + "null", + "integer" + ] }, "width": { - "type": "integer" + "type": [ + "null", + "integer" + ] }, "y": { - "type": "integer" + "type": [ + "null", + "integer" + ] }, "height": { - "type": "integer" + "type": [ + "null", + "integer" + ] } } } } }, "defaultLocale": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": "string" + "type": [ + "null", + "string" + ] }, "language": { - "type": "string" + "type": [ + "null", + "string" + ] } } }, "organizationType": { - "type": "string" + "type": [ + "null", + "string" + ] }, "alternativeNames": { - "type": "array", - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "specialties": { - "type": "array", - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "staffCountRange": { - "type": "string" + "type": [ + "null", + "string" + ] }, "localizedSpecialties": { - "type": "array", - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "industries": { - "type": "array", - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } }, "name": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": "string" + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": "string" + "type": [ + "null", + "string" + ] }, "language": { - "type": "string" + "type": [ + "null", + "string" + ] } } } } }, "primaryOrganizationType": { - "type": "string" + "type": [ + "null", + "string" + ] }, "locations": { - "type": "array", - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "locationType": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "object" + ], + "properties": { + "localized": { + "type": [ + "null", + "object" + ], + "properties": { + "en_US": { + "type": [ + "null", + "string" + ] + } + } + }, + "preferredLocale": { + "type": [ + "null", + "object" + ], + "properties": { + "country": { + "type": [ + "null", + "string" + ] + }, + "language": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "address": { + "type": [ + "null", + "object" + ], + "properties": { + "geographicArea": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "line1": { + "type": [ + "null", + "string" + ] + }, + "postalCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "localizedDescription": { + "type": [ + "null", + "string" + ] + }, + "geoLocation": { + "type": [ + "null", + "string" + ] + }, + "streetAddressFieldState": { + "type": [ + "null", + "string" + ] + } + } + } }, "id": { - "type": "integer" + "type": [ + "null", + "integer" + ] }, "localizedDescription": { - "type": "string" + "type": [ + "null", + "string" + ] }, "$URN": { - "type": "string" + "type": [ + "null", + "string" + ] }, "localizedWebsite": { - "type": "string" + "type": [ + "null", + "string" + ] }, "logoV2": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "cropped": { - "type": "string" + "type": [ + "null", + "string" + ] }, "original": { - "type": "string" + "type": [ + "null", + "string" + ] }, "cropInfo": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "x": { - "type": "integer" + "type": [ + "null", + "integer" + ] }, "width": { - "type": "integer" + "type": [ + "null", + "integer" + ] }, "y": { - "type": "integer" + "type": [ + "null", + "integer" + ] }, "height": { - "type": "integer" + "type": [ + "null", + "integer" + ] } } } From 2dfe6b15f206a6b50f0a158eaec788904b3fa498 Mon Sep 17 00:00:00 2001 From: Jordan Scott Date: Wed, 1 Jun 2022 13:44:11 -0500 Subject: [PATCH 06/21] added nulls to share_statistics schema --- .../schemas/share_statistics.json | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json index 2bea295e87bf9..621997bd13b52 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json @@ -1,27 +1,26 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", + "type": ["null", "object"], "properties": { "paging": { - "type": "object", + "type": ["null", "object"], "properties": { "start": { - "type": "integer" + "type": ["null", "integer"] }, "count": { - "type": "integer" + "type": ["null", "integer"] }, "links": { - "type": "array", + "type": ["null", "array"], "items": {} }, "total": { - "type": "integer" + "type": ["null", "integer"] } } }, "elements": { - "type": "array", + "type": ["null", "array"], "items": {} } } From de462658078fc31f9e5845407a04d7bc5858c3be Mon Sep 17 00:00:00 2001 From: Jordan Scott Date: Thu, 19 May 2022 13:56:10 -0500 Subject: [PATCH 07/21] added linkedin pages source connector --- .../source-linkedin-pages/.dockerignore | 6 + .../source-linkedin-pages/Dockerfile | 38 + .../source-linkedin-pages/README.md | 132 ++ .../acceptance-test-config.yml | 30 + .../acceptance-test-docker.sh | 16 + .../source-linkedin-pages/bootstrap.md | 7 + .../source-linkedin-pages/build.gradle | 9 + .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 23 + .../integration_tests/acceptance.py | 13 + .../integration_tests/configured_catalog.json | 67 + .../integration_tests/invalid_config.json | 7 + .../integration_tests/sample_config.json | 7 + .../integration_tests/sample_state.json | 5 + .../connectors/source-linkedin-pages/main.py | 12 + .../source-linkedin-pages/requirements.txt | 2 + .../connectors/source-linkedin-pages/setup.py | 30 + .../source_linkedin_pages/__init__.py | 8 + .../source_linkedin_pages/analytics.py | 185 ++ .../schemas/follower_statistics.json | 201 ++ .../schemas/organization_lookup.json | 272 +++ .../schemas/page_statistics.json | 1726 +++++++++++++++++ .../schemas/share_statistics.json | 68 + .../source_linkedin_pages/schemas/shares.json | 109 ++ .../schemas/total_follower_count.json | 8 + .../schemas/ugc_posts.json | 302 +++ .../source_linkedin_pages/source.py | 166 ++ .../source_linkedin_pages/spec.json | 78 + .../source_linkedin_pages/utils.py | 324 ++++ .../unit_tests/__init__.py | 3 + .../unit_tests/test_incremental_streams.py | 59 + .../unit_tests/test_source.py | 22 + .../unit_tests/test_streams.py | 83 + 33 files changed, 4021 insertions(+) create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/.dockerignore create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/Dockerfile create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/README.md create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/bootstrap.md create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/build.gradle create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/main.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/requirements.txt create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/setup.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/__init__.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/analytics.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_incremental_streams.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-linkedin-pages/.dockerignore b/airbyte-integrations/connectors/source-linkedin-pages/.dockerignore new file mode 100644 index 0000000000000..53de6536d1ff3 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_linkedin_pages +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile b/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile new file mode 100644 index 0000000000000..6eff6390b8787 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.7.11-alpine3.14 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_linkedin_pages ./source_linkedin_pages + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-linkedin-pages diff --git a/airbyte-integrations/connectors/source-linkedin-pages/README.md b/airbyte-integrations/connectors/source-linkedin-pages/README.md new file mode 100644 index 0000000000000..5671f959d3ff5 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/README.md @@ -0,0 +1,132 @@ +# Linkedin Pages Source + +This is the repository for the Linkedin Pages source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/linkedin-pages). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-linkedin-pages:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/linkedin-pages) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_linkedin_pages/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source linkedin-pages test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-linkedin-pages:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-linkedin-pages:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-linkedin-pages:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-pages:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-pages:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-linkedin-pages:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-linkedin-pages:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-linkedin-pages:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml new file mode 100644 index 0000000000000..15191f1b4d4d6 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-linkedin-pages:dev0.1.1 +tests: + spec: + - spec_path: "source_linkedin_pages/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + # incremental: # TODO if your connector does not implement incremental sync, remove this block + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-linkedin-pages/bootstrap.md b/airbyte-integrations/connectors/source-linkedin-pages/bootstrap.md new file mode 100644 index 0000000000000..90e5cbd5eb5c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/bootstrap.md @@ -0,0 +1,7 @@ +The LinkedIn Marketing Developer Platform API can be used to pull data from LinkedIn Organizations such as company page details, follower counts, follower statistics, and all sorts of organic content data. + +You must have a LinkedIn Developers' App created in order to request access to the Marketing Developer Platform API. API Access approval takes up to 72 hours after submitting a ~15-question form. + +The app also must be verified by an admin of the LinkedIn organization your app is created for. Once the app is "verified" and granted access to the Marketing Developer Platform API, you can use their easy-peasy OAuth Token Tools to generate access tokens **and** refresh tokens. + +You can access the `client id` and `client secret` in the **Auth** tab of the app dashboard to round out all of the authorization needs you may have. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/build.gradle b/airbyte-integrations/connectors/source-linkedin-pages/build.gradle new file mode 100644 index 0000000000000..df9098d6060fc --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_linkedin_pages' +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/__init__.py b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..ceae7331d2d9a --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json @@ -0,0 +1,23 @@ +{ + "organization_lookup": { + "lastModified": "2050-01-01" + }, + "follower_statistics": { + "lastModified": "2050-01-01" + }, + "page_statistics": { + "lastModified": "2050-01-01" + }, + "share_statistics": { + "lastModified": "2050-01-01" + }, + "shares": { + "lastModified": "2050-01-01" + }, + "total_follower_count": { + "end_date": "2050-01-01" + }, + "ugc_posts": { + "end_date": "2050-01-01" + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py new file mode 100644 index 0000000000000..1b85ccdcde521 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + yield \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..6d5c31ddc7549 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json @@ -0,0 +1,67 @@ +{ + "streams": [ + { + "stream": { + "name": "organization_lookup", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "follower_statistics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "page_statistics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "share_statistics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "shares", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "total_follower_count", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "ugc_posts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..5c8c7b29a99f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/invalid_config.json @@ -0,0 +1,7 @@ +{ + "org_id": 12345678, + "credentials": { + "auth_method": "access_token", + "access_token": "wrong_token_sra6ibiw0ZWEdMnC0ZizeD1gLRQP6u1pkQl" + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json new file mode 100644 index 0000000000000..c06b53f2c96e3 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json @@ -0,0 +1,7 @@ +{ + "org_id": 12345678, + "credentials": { + "auth_method": "access_token", + "access_token": "drAL4JXBJ6v1qaU9iWargosra6ibiw0ZWEdMnC0ZizeD1gLRQP6u1pkQ" + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_state.json new file mode 100644 index 0000000000000..3587e579822d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/main.py b/airbyte-integrations/connectors/source-linkedin-pages/main.py new file mode 100644 index 0000000000000..d601480ca89df --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/main.py @@ -0,0 +1,12 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import sys + +from airbyte_cdk.entrypoint import launch +from source_linkedin_pages import SourceLinkedinPages + +if __name__ == "__main__": + source = SourceLinkedinPages() + launch(source, sys.argv[1:]) \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/requirements.txt b/airbyte-integrations/connectors/source-linkedin-pages/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-linkedin-pages/setup.py b/airbyte-integrations/connectors/source-linkedin-pages/setup.py new file mode 100644 index 0000000000000..bae2e6204f394 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/setup.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", + "pendulum~=2.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_linkedin_pages", + description="Source implementation for Linkedin Company Pages.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/__init__.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/__init__.py new file mode 100644 index 0000000000000..e4157287bd16d --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceLinkedinPages + +__all__ = ["SourceLinkedinPages"] diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/analytics.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/analytics.py new file mode 100644 index 0000000000000..9cbdc2d307919 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/analytics.py @@ -0,0 +1,185 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from collections import defaultdict +from typing import Any, Iterable, List, Mapping + +import pendulum as pdm + +from .utils import get_parent_stream_values + +# LinkedIn has a max of 20 fields per request. We make chunks by size of 17 fields +# to have the `dateRange`, `pivot`, and `pivotValue` be included as well. +FIELDS_CHUNK_SIZE = 17 +# Number of days ahead for date slices, from start date. +WINDOW_IN_DAYS = 30 +# List of adAnalyticsV2 fields available for fetch +ANALYTICS_FIELDS_V2: List = [ + "actionClicks", + "adUnitClicks", + "approximateUniqueImpressions", + "cardClicks", + "cardImpressions", + "clicks", + "commentLikes", + "comments", + "companyPageClicks", + "conversionValueInLocalCurrency", + "costInLocalCurrency", + "costInUsd", + "dateRange", + "externalWebsiteConversions", + "externalWebsitePostClickConversions", + "externalWebsitePostViewConversions", + "follows", + "fullScreenPlays", + "impressions", + "landingPageClicks", + "leadGenerationMailContactInfoShares", + "leadGenerationMailInterestedClicks", + "likes", + "oneClickLeadFormOpens", + "oneClickLeads", + "opens", + "otherEngagements", + "pivot", + "pivotValue", + "pivotValues", + "reactions", + "sends", + "shares", + "textUrlClicks", + "totalEngagements", + "videoCompletions", + "videoFirstQuartileCompletions", + "videoMidpointCompletions", + "videoStarts", + "videoThirdQuartileCompletions", + "videoViews", + "viralCardClicks", + "viralCardImpressions", + "viralClicks", + "viralCommentLikes", + "viralComments", + "viralCompanyPageClicks", + "viralExternalWebsiteConversions", + "viralExternalWebsitePostClickConversions", + "viralExternalWebsitePostViewConversions", + "viralFollows", + "viralFullScreenPlays", + "viralImpressions", + "viralLandingPageClicks", + "viralLikes", + "viralOneClickLeadFormOpens", + "viralOneClickLeads", + "viralOtherEngagements", + "viralReactions", + "viralShares", + "viralTotalEngagements", + "viralVideoCompletions", + "viralVideoFirstQuartileCompletions", + "viralVideoMidpointCompletions", + "viralVideoStarts", + "viralVideoThirdQuartileCompletions", + "viralVideoViews", +] + +# Fields that are always present in fields_set chunks +BASE_ANALLYTICS_FIELDS = ["dateRange", "pivot", "pivotValue"] + + +def chunk_analytics_fields( + fields: List = ANALYTICS_FIELDS_V2, + base_fields: List = BASE_ANALLYTICS_FIELDS, + fields_chunk_size: int = FIELDS_CHUNK_SIZE, +) -> Iterable[List]: + """ + Chunks the list of available fields into the chunks of equal size. + """ + # Make chunks + chunks = list((fields[f : f + fields_chunk_size] for f in range(0, len(fields), fields_chunk_size))) + # Make sure base_fields are within the chunks + for chunk in chunks: + for field in base_fields: + if field not in chunk: + chunk.append(field) + yield from chunks + + +def make_date_slices(start_date: str, end_date: str = None, window_in_days: int = WINDOW_IN_DAYS) -> Iterable[List]: + """ + Produces date slices from start_date to end_date (if specified), + otherwise end_date will be present time. + """ + start = pdm.parse(start_date) + end = pdm.parse(end_date) if end_date else pdm.now() + date_slices = [] + while start < end: + slice_end_date = start.add(days=window_in_days) + date_slice = { + "start.day": start.day, + "start.month": start.month, + "start.year": start.year, + "end.day": slice_end_date.day, + "end.month": slice_end_date.month, + "end.year": slice_end_date.year, + } + date_slices.append({"dateRange": date_slice}) + start = slice_end_date + yield from date_slices + + +def make_analytics_slices( + record: Mapping[str, Any], key_value_map: Mapping[str, Any], start_date: str, end_date: str = None +) -> Iterable[Mapping[str, Any]]: + """ + We drive the ability to directly pass the prepared parameters inside the stream_slice. + The output of this method is ready slices for analytics streams: + """ + # define the base_slice + base_slice = get_parent_stream_values(record, key_value_map) + # add chunked fields, date_slices to the base_slice + analytics_slices = [] + for fields_set in chunk_analytics_fields(): + base_slice["fields"] = ",".join(map(str, fields_set)) + for date_slice in make_date_slices(start_date, end_date): + base_slice.update(**date_slice) + analytics_slices.append(base_slice.copy()) + yield from analytics_slices + + +def update_analytics_params(stream_slice: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Produces the date range parameters from input stream_slice + """ + return { + # Start date range + "dateRange.start.day": stream_slice["dateRange"]["start.day"], + "dateRange.start.month": stream_slice["dateRange"]["start.month"], + "dateRange.start.year": stream_slice["dateRange"]["start.year"], + # End date range + "dateRange.end.day": stream_slice["dateRange"]["end.day"], + "dateRange.end.month": stream_slice["dateRange"]["end.month"], + "dateRange.end.year": stream_slice["dateRange"]["end.year"], + # Chunk of fields + "fields": stream_slice["fields"], + } + + +def merge_chunks(chunked_result: Iterable[Mapping[str, Any]], merge_by_key: str) -> Iterable[Mapping[str, Any]]: + """ + We need to merge the chunked API responses + into the single structure using any available unique field. + """ + # Merge the pieces together + merged = defaultdict(dict) + for chunk in chunked_result: + for item in chunk: + merged[item[merge_by_key]].update(item) + # Clean up the result by getting out the values of the merged keys + result = [] + for item in merged: + result.append(merged.get(item)) + yield from result \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json new file mode 100644 index 0000000000000..1e1d0298f3d93 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -0,0 +1,201 @@ +{ + "type": "object", + "properties": { + "paging": { + "type": "object", + "properties": { + "start": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "links": { + "type": "array", + "items": {} + } + } + }, + "elements": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "followerCountsByAssociationType": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "followerCounts": { + "type": "object", + "properties": { + "organicFollowerCount": { + "type": "integer" + }, + "paidFollowerCount": { + "type": "integer" + } + } + }, + "associationType": { + "type": "string" + } + } + } + ] + }, + "followerCountsByRegion": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "region": { + "type": "string" + }, + "followerCounts": { + "type": "object", + "properties": { + "organicFollowerCount": { + "type": "integer" + }, + "paidFollowerCount": { + "type": "integer" + } + } + } + } + } + ] + }, + "followerCountsBySeniority": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "followerCounts": { + "type": "object", + "properties": { + "organicFollowerCount": { + "type": "integer" + }, + "paidFollowerCount": { + "type": "integer" + } + } + }, + "seniority": { + "type": "string" + } + } + } + ] + }, + "followerCountsByIndustry": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "followerCounts": { + "type": "object", + "properties": { + "organicFollowerCount": { + "type": "integer" + }, + "paidFollowerCount": { + "type": "integer" + } + } + }, + "industry": { + "type": "string" + } + } + } + ] + }, + "followerCountsByFunction": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "followerCounts": { + "type": "object", + "properties": { + "organicFollowerCount": { + "type": "integer" + }, + "paidFollowerCount": { + "type": "integer" + } + } + }, + "function": { + "type": "string" + } + } + } + ] + }, + "followerCountsByStaffCountRange": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "followerCounts": { + "type": "object", + "properties": { + "organicFollowerCount": { + "type": "integer" + }, + "paidFollowerCount": { + "type": "integer" + } + } + }, + "staffCountRange": { + "type": "string" + } + } + } + ] + }, + "followerCountsByCountry": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "followerCounts": { + "type": "object", + "properties": { + "organicFollowerCount": { + "type": "integer" + }, + "paidFollowerCount": { + "type": "integer" + } + } + }, + "country": { + "type": "string" + } + } + } + ] + }, + "organizationalEntity": { + "type": "string" + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json new file mode 100644 index 0000000000000..11e1446820854 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -0,0 +1,272 @@ +{ + "type": "object", + "properties": { + "vanityName": { + "type": "string" + }, + "localizedName": { + "type": "string" + }, + "website": { + "type": "object", + "properties": { + "localized": { + "type": "object", + "properties": { + "en_US": { + "type": "string" + } + } + }, + "preferredLocale": { + "type": "object", + "properties": { + "country": { + "type": "string" + }, + "language": { + "type": "string" + } + } + } + } + }, + "foundedOn": { + "type": "object", + "properties": { + "year": { + "type": "integer" + } + } + }, + "groups": { + "type": "array", + "items": {} + }, + "description": { + "type": "object", + "properties": { + "localized": { + "type": "object", + "properties": { + "en_US": { + "type": "string" + } + } + }, + "preferredLocale": { + "type": "object", + "properties": { + "country": { + "type": "string" + }, + "language": { + "type": "string" + } + } + } + } + }, + "versionTag": { + "type": "string" + }, + "coverPhotoV2": { + "type": "object", + "properties": { + "cropped": { + "type": "string" + }, + "original": { + "type": "string" + }, + "cropInfo": { + "type": "object", + "properties": { + "x": { + "type": "integer" + }, + "width": { + "type": "integer" + }, + "y": { + "type": "integer" + }, + "height": { + "type": "integer" + } + } + } + } + }, + "defaultLocale": { + "type": "object", + "properties": { + "country": { + "type": "string" + }, + "language": { + "type": "string" + } + } + }, + "organizationType": { + "type": "string" + }, + "alternativeNames": { + "type": "array", + "items": {} + }, + "specialties": { + "type": "array", + "items": {} + }, + "staffCountRange": { + "type": "string" + }, + "localizedSpecialties": { + "type": "array", + "items": {} + }, + "industries": { + "type": "array", + "items": [ + { + "type": "string" + } + ] + }, + "name": { + "type": "object", + "properties": { + "localized": { + "type": "object", + "properties": { + "en_US": { + "type": "string" + } + } + }, + "preferredLocale": { + "type": "object", + "properties": { + "country": { + "type": "string" + }, + "language": { + "type": "string" + } + } + } + } + }, + "primaryOrganizationType": { + "type": "string" + }, + "locations": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "locationType": { + "type": "string" + }, + "description": { + "type": "object", + "properties": { + "localized": { + "type": "object", + "properties": { + "en_US": { + "type": "string" + } + } + }, + "preferredLocale": { + "type": "object", + "properties": { + "country": { + "type": "string" + }, + "language": { + "type": "string" + } + } + } + } + }, + "address": { + "type": "object", + "properties": { + "geographicArea": { + "type": "string" + }, + "country": { + "type": "string" + }, + "city": { + "type": "string" + }, + "line1": { + "type": "string" + }, + "postalCode": { + "type": "string" + } + } + }, + "localizedDescription": { + "type": "string" + }, + "geoLocation": { + "type": "string" + }, + "streetAddressFieldState": { + "type": "string" + } + } + } + ] + }, + "id": { + "type": "integer" + }, + "localizedDescription": { + "type": "string" + }, + "$URN": { + "type": "string" + }, + "localizedWebsite": { + "type": "string" + }, + "logoV2": { + "type": "object", + "properties": { + "cropped": { + "type": "string" + }, + "original": { + "type": "string" + }, + "cropInfo": { + "type": "object", + "properties": { + "x": { + "type": "integer" + }, + "width": { + "type": "integer" + }, + "y": { + "type": "integer" + }, + "height": { + "type": "integer" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json new file mode 100644 index 0000000000000..a003e0efc4bc0 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json @@ -0,0 +1,1726 @@ +{ + "type": "object", + "properties": { + "paging": { + "type": "object", + "properties": { + "start": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "links": { + "type": "array", + "items": {} + } + } + }, + "elements": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "pageStatisticsBySeniority": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "pageStatistics": { + "type": "object", + "properties": { + "views": { + "type": "object", + "properties": { + "mobileProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allDesktopPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "insightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allMobilePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "jobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "productsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "peoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "overviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "lifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "careersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopPeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "aboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobilePeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + } + } + } + } + }, + "seniority": { + "type": "string" + } + } + } + ] + }, + "pageStatisticsByCountry": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "country": { + "type": "string" + }, + "pageStatistics": { + "type": "object", + "properties": { + "views": { + "type": "object", + "properties": { + "mobileProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allDesktopPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "insightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allMobilePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "jobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "productsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "peoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "overviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "lifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "careersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopPeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "aboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobilePeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + } + } + } + } + } + } + } + ] + }, + "pageStatisticsByIndustry": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "pageStatistics": { + "type": "object", + "properties": { + "views": { + "type": "object", + "properties": { + "mobileProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allDesktopPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "insightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allMobilePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "jobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "productsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "peoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "overviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "lifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "careersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopPeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "aboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobilePeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + } + } + } + } + } + } + } + ] + }, + "organization": { + "type": "string" + }, + "totalPageStatistics": { + "type": "object", + "properties": { + "clicks": { + "type": "object", + "properties": { + "mobileCareersPageClicks": { + "type": "object", + "properties": { + "careersPageJobsClicks": { + "type": "integer" + }, + "careersPagePromoLinksClicks": { + "type": "integer" + }, + "careersPageEmployeesClicks": { + "type": "integer" + } + } + }, + "careersPageClicks": { + "type": "object", + "properties": { + "careersPagePromoLinksClicks": { + "type": "integer" + }, + "careersPageBannerPromoClicks": { + "type": "integer" + }, + "careersPageJobsClicks": { + "type": "integer" + }, + "careersPageEmployeesClicks": { + "type": "integer" + } + } + } + } + }, + "views": { + "type": "object", + "properties": { + "mobileProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allDesktopPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "insightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allMobilePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "jobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "productsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "peoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "overviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "lifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "careersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopPeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "aboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobilePeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + } + } + } + } + }, + "pageStatisticsByStaffCountRange": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "staffCountRange": { + "type": "string" + }, + "pageStatistics": { + "type": "object", + "properties": { + "views": { + "type": "object", + "properties": { + "mobileProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allDesktopPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "insightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allMobilePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "jobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "productsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "peoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "overviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "lifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "careersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopPeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "aboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobilePeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + } + } + } + } + } + } + } + ] + }, + "pageStatisticsByRegion": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "region": { + "type": "string" + }, + "pageStatistics": { + "type": "object", + "properties": { + "views": { + "type": "object", + "properties": { + "mobileProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allDesktopPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "insightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allMobilePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "jobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "productsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "peoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "overviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "lifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "careersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopPeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "aboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobilePeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + } + } + } + } + } + } + } + ] + }, + "pageStatisticsByFunction": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "pageStatistics": { + "type": "object", + "properties": { + "views": { + "type": "object", + "properties": { + "mobileProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allDesktopPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "insightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allMobilePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "jobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "productsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopProductsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "peoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "overviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "lifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopOverviewPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "allPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "careersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopJobsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopPeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "aboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopAboutPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobilePeoplePageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopCareersPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "desktopLifeAtPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + }, + "mobileInsightsPageViews": { + "type": "object", + "properties": { + "pageViews": { + "type": "integer" + } + } + } + } + } + } + }, + "function": { + "type": "string" + } + } + } + ] + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json new file mode 100644 index 0000000000000..dfc9a7e6a7115 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json @@ -0,0 +1,68 @@ +{ + "type": "object", + "properties": { + "paging": { + "type": "object", + "properties": { + "start": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "links": { + "type": "array", + "items": {} + }, + "total": { + "type": "integer" + } + } + }, + "elements": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "totalShareStatistics": { + "type": "object", + "properties": { + "uniqueImpressionsCount": { + "type": "integer" + }, + "shareCount": { + "type": "integer" + }, + "shareMentionsCount": { + "type": "integer" + }, + "engagement": { + "type": "number" + }, + "clickCount": { + "type": "integer" + }, + "likeCount": { + "type": "integer" + }, + "impressionCount": { + "type": "integer" + }, + "commentMentionsCount": { + "type": "integer" + }, + "commentCount": { + "type": "integer" + } + } + }, + "organizationalEntity": { + "type": "string" + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json new file mode 100644 index 0000000000000..5e0fd75a811ce --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json @@ -0,0 +1,109 @@ +{ + "type": "object", + "properties": { + "paging": { + "type": "object", + "properties": { + "start": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "links": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "rel": { + "type": "string" + }, + "href": { + "type": "string" + } + } + } + ] + }, + "total": { + "type": "integer" + } + } + }, + "elements": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "owner": { + "type": "string" + }, + "activity": { + "type": "string" + }, + "edited": { + "type": "boolean" + }, + "resharedShare": { + "type": "string" + }, + "created": { + "type": "object", + "properties": { + "actor": { + "type": "string" + }, + "time": { + "type": "integer" + } + } + }, + "lastModified": { + "type": "object", + "properties": { + "actor": { + "type": "string" + }, + "time": { + "type": "integer" + } + } + }, + "text": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + }, + "id": { + "type": "string" + }, + "originalShare": { + "type": "string" + }, + "distribution": { + "type": "object", + "properties": { + "linkedInDistributionTarget": { + "type": "object", + "properties": { + "visibleToGuest": { + "type": "boolean" + } + } + } + } + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json new file mode 100644 index 0000000000000..18f07c4bbb778 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json @@ -0,0 +1,8 @@ +{ + "type": "object", + "properties": { + "firstDegreeSize": { + "type": "integer" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json new file mode 100644 index 0000000000000..f95bb41047b7e --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json @@ -0,0 +1,302 @@ +{ + "type": "object", + "properties": { + "paging": { + "type": "object", + "properties": { + "start": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "links": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "rel": { + "type": "string" + }, + "href": { + "type": "string" + } + } + } + ] + }, + "total": { + "type": "integer" + } + } + }, + "elements": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "lifecycleState": { + "type": "string" + }, + "specificContent": { + "type": "object", + "properties": { + "com.linkedin.ugc.ShareContent": { + "type": "object", + "properties": { + "shareCommentary": { + "type": "object", + "properties": { + "inferredLocale": { + "type": "string" + }, + "attributes": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "length": { + "type": "integer" + }, + "start": { + "type": "integer" + }, + "value": { + "type": "object", + "properties": { + "com.linkedin.common.MemberAttributedEntity": { + "type": "object", + "properties": { + "member": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "length": { + "type": "integer" + }, + "start": { + "type": "integer" + }, + "value": { + "type": "object", + "properties": { + "com.linkedin.common.HashtagAttributedEntity": { + "type": "object", + "properties": { + "hashtag": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "length": { + "type": "integer" + }, + "start": { + "type": "integer" + }, + "value": { + "type": "object", + "properties": { + "com.linkedin.common.HashtagAttributedEntity": { + "type": "object", + "properties": { + "hashtag": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "length": { + "type": "integer" + }, + "start": { + "type": "integer" + }, + "value": { + "type": "object", + "properties": { + "com.linkedin.common.HashtagAttributedEntity": { + "type": "object", + "properties": { + "hashtag": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "length": { + "type": "integer" + }, + "start": { + "type": "integer" + }, + "value": { + "type": "object", + "properties": { + "com.linkedin.common.HashtagAttributedEntity": { + "type": "object", + "properties": { + "hashtag": { + "type": "string" + } + } + } + } + } + } + } + ] + }, + "text": { + "type": "string" + } + } + }, + "media": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "thumbnails": { + "type": "array", + "items": {} + }, + "status": { + "type": "string" + }, + "media": { + "type": "string" + } + } + } + ] + }, + "shareFeatures": { + "type": "object", + "properties": { + "hashtags": { + "type": "array", + "items": [ + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + } + ] + } + } + }, + "shareMediaCategory": { + "type": "string" + } + } + } + } + }, + "visibility": { + "type": "object", + "properties": { + "com.linkedin.ugc.MemberNetworkVisibility": { + "type": "string" + } + } + }, + "created": { + "type": "object", + "properties": { + "actor": { + "type": "string" + }, + "time": { + "type": "integer" + } + } + }, + "author": { + "type": "string" + }, + "versionTag": { + "type": "string" + }, + "id": { + "type": "string" + }, + "firstPublishedAt": { + "type": "integer" + }, + "lastModified": { + "type": "object", + "properties": { + "actor": { + "type": "string" + }, + "time": { + "type": "integer" + } + } + }, + "distribution": { + "type": "object", + "properties": { + "externalDistributionChannels": { + "type": "array", + "items": {} + }, + "distributedViaFollowFeed": { + "type": "boolean" + }, + "feedDistribution": { + "type": "string" + } + } + }, + "contentCertificationRecord": { + "type": "string" + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py new file mode 100644 index 0000000000000..214923cc0f94c --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py @@ -0,0 +1,166 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator + + +class LinkedinPagesStream(HttpStream, ABC): + + url_base = "https://api.linkedin.com/v2/" + primary_key = None + + def __init__(self, config): + super().__init__(authenticator=config.get("authenticator")) + self.config = config + + + @property + def org(self): + """Property to return the user Organization Id from input""" + return self.config.get("org_id") + + def path(self, **kwargs) -> str: + """Returns the API endpoint path for stream, from `endpoint` class attribute.""" + return self.endpoint + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None + ) -> Iterable[Mapping]: + return [response.json()] + +class OrganizationLookup(LinkedinPagesStream): + + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"organizations/{self.org}" + return path + +class FollowerStatistics(LinkedinPagesStream): + + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"organizationalEntityFollowerStatistics?q=organizationalEntity&organizationalEntity=urn:li:organization:{self.org}" + return path + +class PageStatistics(LinkedinPagesStream): + + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"organizationPageStatistics?q=organization&organization=urn%3Ali%3Aorganization%3A{self.org}" + return path + +class ShareStatistics(LinkedinPagesStream): + + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"organizationalEntityShareStatistics?q=organizationalEntity&organizationalEntity=urn%3Ali%3Aorganization%3A{self.org}" + return path + +class Shares(LinkedinPagesStream): + + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"shares?q=owners&owners=urn%3Ali%3Aorganization%3A{self.org}&sortBy=LAST_MODIFIED&sharesPerOwner=50" + return path + +class TotalFollowerCount(LinkedinPagesStream): + + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"networkSizes/urn:li:organization:{self.org}?edgeType=CompanyFollowedByMember" + return path + +class UgcPosts(LinkedinPagesStream): + + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"ugcPosts?q=authors&authors=List(urn%3Ali%3Aorganization%3A{self.org})&sortBy=LAST_MODIFIED&count=50" + return path + + def request_headers(self, stream_state: Mapping[str, Any], **kwargs) -> Mapping[str, Any]: + """ + If org_ids are specified as user's input from configuration, + we must use MODIFIED header: {'X-RestLi-Protocol-Version': '2.0.0'} + """ + return {"X-RestLi-Protocol-Version": "2.0.0"} if self.org else {} + + +class SourceLinkedinPages(AbstractSource): + """ + Abstract Source inheritance, provides: + - implementation for `check` connector's connectivity + - implementation to call each stream with it's input parameters. + """ + + @classmethod + def get_authenticator(cls, config: Mapping[str, Any]) -> TokenAuthenticator: + """ + Validate input parameters and generate a necessary Authentication object + This connectors support 2 auth methods: + 1) direct access token with TTL = 2 months + 2) refresh token (TTL = 1 year) which can be converted to access tokens + Every new refresh revokes all previous access tokens q + """ + auth_method = config.get("credentials", {}).get("auth_method") + if not auth_method or auth_method == "access_token": + # support of backward compatibility with old exists configs + access_token = config["credentials"]["access_token"] if auth_method else config["access_token"] + return TokenAuthenticator(token=access_token) + elif auth_method == "oAuth2.0": + return Oauth2Authenticator( + token_refresh_endpoint="https://www.linkedin.com/oauth/v2/accessToken", + client_id=config["credentials"]["client_id"], + client_secret=config["credentials"]["client_secret"], + refresh_token=config["credentials"]["refresh_token"], + ) + raise Exception("incorrect input parameters") + + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: + # RUN $ python main.py check --config secrets/config.json + + """ + Testing connection availability for the connector. + :: for this check method the Customer must have the "r_liteprofile" scope enabled. + :: more info: https://docs.microsoft.com/linkedin/consumer/integrations/self-serve/sign-in-with-linkedin + """ + + config["authenticator"] = self.get_authenticator(config) + stream = OrganizationLookup(config) + stream.records_limit = 1 + try: + next(stream.read_records(sync_mode=SyncMode.full_refresh), None) + return True, None + except Exception as e: + return False, e + + # RUN: $ python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + config["authenticator"] = self.get_authenticator(config) + return [ + OrganizationLookup(config), + FollowerStatistics(config), + PageStatistics(config), + ShareStatistics(config), + Shares(config), + TotalFollowerCount(config), + UgcPosts(config) + ] + diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json new file mode 100644 index 0000000000000..be64213a3de8f --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json @@ -0,0 +1,78 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/linkedin-pages/", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Linkedin Pages Spec", + "type": "object", + "required": ["org_id"], + "additionalProperties": true, + "properties": { + "org_id": { + "title": "Organization ID", + "type": "integer", + "description": "Specify the Organization ID", + "examples": [123456789] + }, + "credentials": { + "title": "Authentication *", + "type": "object", + "oneOf": [ + { + "type": "object", + "title": "OAuth2.0", + "required": ["client_id", "client_secret", "refresh_token"], + "properties": { + "auth_method": { + "type": "string", + "const": "oAuth2.0" + }, + "client_id": { + "type": "string", + "title": "Client ID", + "description": "The client ID of the LinkedIn developer application.", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "title": "Client secret", + "description": "The client secret of the LinkedIn developer application.", + "airbyte_secret": true + }, + "refresh_token": { + "type": "string", + "title": "Refresh token", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours.", + "airbyte_secret": true + } + } + }, + { + "title": "Access token", + "type": "object", + "required": ["access_token"], + "properties": { + "auth_method": { + "type": "string", + "const": "access_token" + }, + "access_token": { + "type": "string", + "title": "Access token", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours.", + "airbyte_secret": true + } + } + } + ] + } + } + }, + "authSpecification": { + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": ["credentials", "0"], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["refresh_token"]] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py new file mode 100644 index 0000000000000..745ed95943ea2 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py @@ -0,0 +1,324 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import json +from typing import Any, Dict, Iterable, List, Mapping + +import pendulum as pdm + + +def get_parent_stream_values(record: Dict, key_value_map: Dict) -> Dict: + """ + Outputs the Dict with key:value slices for the stream. + :: EXAMPLE: + Input: + records = [{dict}, {dict}, ...], + key_value_map = {: } + + Output: + { + : records..value, + } + """ + result = {} + for key in key_value_map: + value = record.get(key_value_map[key]) + if value: + result[key] = value + return result + + +def transform_change_audit_stamps( + record: Dict, dict_key: str = "changeAuditStamps", props: List = ["created", "lastModified"], fields: List = ["time"] +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `changeAuditStamps` input structure: + { + "changeAuditStamps": { + "created": {"time": 1629581275000}, + "lastModified": {"time": 1629664544760} + } + } + + :: EXAMPLE output: + { + "created": "2021-08-21 21:27:55", + "lastModified": "2021-08-22 20:35:44" + } + """ + + target_dict: Dict = record.get(dict_key) + for prop in props: + # Update dict with flatten key:value + for field in fields: + record[prop] = pdm.from_timestamp(target_dict.get(prop).get(field) / 1000).to_datetime_string() + record.pop(dict_key) + + return record + + +def date_str_from_date_range(record: Dict, prefix: str) -> str: + """ + Makes the ISO8601 format date string from the input . + + EXAMPLE: + Input: record + { + "start.year": 2021, "start.month": 8, "start.day": 1, + "end.year": 2021, "end.month": 9, "end.day": 31 + } + + EXAMPLE output: + With `prefix` = "start" + str: "2021-08-13", + + With `prefix` = "end" + str: "2021-09-31", + """ + + year = record.get(f"{prefix}.year") + month = record.get(f"{prefix}.month") + day = record.get(f"{prefix}.day") + return pdm.date(year, month, day).to_date_string() + + +def transform_date_range( + record: Dict, + dict_key: str = "dateRange", + props: List = ["start", "end"], + fields: List = ["year", "month", "day"], +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `dateRange` input structure in Analytics streams: + { + "dateRange": { + "start": {"month": 8, "day": 13, "year": 2021}, + "end": {"month": 8, "day": 13, "year": 2021} + } + } + :: EXAMPLE output: + { + "start_date": "2021-08-13", + "end_date": "2021-08-13" + } + """ + # define list of tmp keys for cleanup. + keys_to_remove = [dict_key, "start.day", "start.month", "start.year", "end.day", "end.month", "end.year", "start", "end"] + + target_dict: Dict = record.get(dict_key) + for prop in props: + # Update dict with flatten key:value + for field in fields: + record.update(**{f"{prop}.{field}": target_dict.get(prop).get(field)}) + # We build `start_date` & `end_date` fields from nested structure. + record.update(**{"start_date": date_str_from_date_range(record, "start"), "end_date": date_str_from_date_range(record, "end")}) + # Cleanup tmp fields & nested used parts + for key in keys_to_remove: + if key in record: + record.pop(key) + return record + + +def transform_targeting_criteria( + record: Dict, + dict_key: str = "targetingCriteria", +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `targetingCriteria` input structure: + { + "targetingCriteria": { + "include": { + "and": [ + { + "or": { + "urn:li:adTargetingFacet:titles": [ + "urn:li:title:100", + "urn:li:title:10326", + "urn:li:title:10457", + "urn:li:title:10738", + "urn:li:title:10966", + "urn:li:title:11349", + "urn:li:title:1159", + ] + } + }, + {"or": {"urn:li:adTargetingFacet:locations": ["urn:li:geo:103644278"]}}, + {"or": {"urn:li:adTargetingFacet:interfaceLocales": ["urn:li:locale:en_US"]}}, + ] + }, + "exclude": { + "or": { + "urn:li:adTargetingFacet:facet_Key1": [ + "facet_test1", + "facet_test2", + ], + "urn:li:adTargetingFacet:facet_Key2": [ + "facet_test3", + "facet_test4", + ], + } + } + } + + :: EXAMPLE output: + { + "targetingCriteria": { + "include": { + "and": [ + { + "type": "urn:li:adTargetingFacet:titles", + "values": [ + "urn:li:title:100", + "urn:li:title:10326", + "urn:li:title:10457", + "urn:li:title:10738", + "urn:li:title:10966", + "urn:li:title:11349", + "urn:li:title:1159", + ], + }, + { + "type": "urn:li:adTargetingFacet:locations", + "values": ["urn:li:geo:103644278"], + }, + { + "type": "urn:li:adTargetingFacet:interfaceLocales", + "values": ["urn:li:locale:en_US"], + }, + ] + }, + "exclude": { + "or": [ + { + "type": "urn:li:adTargetingFacet:facet_Key1", + "values": ["facet_test1", "facet_test2"], + }, + { + "type": "urn:li:adTargetingFacet:facet_Key2", + "values": ["facet_test3", "facet_test4"], + }, + ] + }, + } + + """ + + def unnest_dict(nested_dict: Dict) -> Iterable[Dict]: + """ + Unnest the nested dict to simplify the normalization + + EXAMPLE OUTPUT: + [ + {"type": "some_key", "values": "some_values"}, + ..., + {"type": "some_other_key", "values": "some_other_values"} + ] + """ + + for key, value in nested_dict.items(): + values = [] + if isinstance(value, List): + if len(value) > 0: + if isinstance(value[0], str): + values = value + elif isinstance(value[0], Dict): + for v in value: + values.append(v) + elif isinstance(value, Dict): + values.append(value) + yield {"type": key, "values": values} + + # get the target dict from record + targeting_criteria = record.get(dict_key) + + # transform `include` + if "include" in targeting_criteria: + and_list = targeting_criteria.get("include").get("and") + updated_include = {"and": []} + for k in and_list: + or_dict = k.get("or") + for j in unnest_dict(or_dict): + updated_include["and"].append(j) + # Replace the original 'and' with updated_include + record["targetingCriteria"]["include"] = updated_include + + # transform `exclude` if present + if "exclude" in targeting_criteria: + or_dict = targeting_criteria.get("exclude").get("or") + updated_exclude = {"or": []} + for k in unnest_dict(or_dict): + updated_exclude["or"].append(k) + # Replace the original 'or' with updated_exclude + record["targetingCriteria"]["exclude"] = updated_exclude + + return record + + +def transform_variables( + record: Dict, + dict_key: str = "variables", +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `variables` input: + { + "variables": { + "data": { + "com.linkedin.ads.SponsoredUpdateCreativeVariables": { + "activity": "urn:li:activity:1234", + "directSponsoredContent": 0, + "share": "urn:li:share:1234", + } + } + } + } + + :: EXAMPLE output: + { + "variables": { + "type": "com.linkedin.ads.SponsoredUpdateCreativeVariables", + "values": [ + {"key": "activity", "value": "urn:li:activity:1234"}, + {"key": "directSponsoredContent", "value": 0}, + {"key": "share", "value": "urn:li:share:1234"}, + ], + } + } + """ + + variables = record.get(dict_key).get("data") + for key, params in variables.items(): + record["variables"]["type"] = key + record["variables"]["values"] = [] + for key, value in params.items(): + # convert various datatypes of values into the string + record["variables"]["values"].append({"key": key, "value": json.dumps(value, ensure_ascii=True)}) + # Clean the nested structure + record["variables"].pop("data") + return record + + +def transform_data(records: List) -> Iterable[Mapping]: + """ + We need to transform the nested complex data structures into simple key:value pair, + to be properly normalised in the destination. + """ + for record in records: + + if "changeAuditStamps" in record: + record = transform_change_audit_stamps(record) + + if "dateRange" in record: + record = transform_date_range(record) + + if "targetingCriteria" in record: + record = transform_targeting_criteria(record) + + if "variables" in record: + record = transform_variables(record) + + yield record \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/__init__.py b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_incremental_streams.py new file mode 100644 index 0000000000000..4b7913646e2ac --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_incremental_streams.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_linkedin_pages.source import IncrementalLinkedinPagesStream + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalLinkedinPagesStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalLinkedinPagesStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalLinkedinPagesStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalLinkedinPagesStream() + # TODO: replace this with your expected cursor field + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class): + stream = IncrementalLinkedinPagesStream() + # TODO: replace this with your input parameters + inputs = {"current_stream_state": None, "latest_record": None} + # TODO: replace this with your expected updated stream state + expected_state = {} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalLinkedinPagesStream() + # TODO: replace this with your input parameters + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + # TODO: replace this with your expected stream slices list + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalLinkedinPagesStream, "cursor_field", "dummy_field") + stream = IncrementalLinkedinPagesStream() + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalLinkedinPagesStream() + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalLinkedinPagesStream() + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py new file mode 100644 index 0000000000000..7f23f89ed1806 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_linkedin_pages.source import SourceLinkedinPages + + +def test_check_connection(mocker): + source = SourceLinkedinPages() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceLinkedinPages() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_streams.py new file mode 100644 index 0000000000000..203fc70bb3dfa --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_streams.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_linkedin_pages.source import LinkedinPagesStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(LinkedinPagesStream, "path", "v0/example_endpoint") + mocker.patch.object(LinkedinPagesStream, "primary_key", "test_primary_key") + mocker.patch.object(LinkedinPagesStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = LinkedinPagesStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = LinkedinPagesStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected next page token + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = LinkedinPagesStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = LinkedinPagesStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = LinkedinPagesStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = LinkedinPagesStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = LinkedinPagesStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time From 13716320cd737952fc3cd9f1b12daa3978b0b284 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Wed, 10 Aug 2022 17:24:31 -0300 Subject: [PATCH 08/21] solve conflict in readme --- .vscode/settings.json | 7 +- CODE_OF_CONDUCT 2.md | 2 + CONTRIBUTING 2.md | 2 + airbyte-integrations/builds.md | 1 + .../source-linkedin-pages/Dockerfile | 2 +- .../acceptance-test-config.yml | 15 +- .../schemas/follower_statistics.json | 191 +- .../schemas/organization_lookup.json | 182 +- .../schemas/page_statistics.json | 1716 +---------------- .../schemas/share_statistics.json | 44 +- .../source_linkedin_pages/schemas/shares.json | 100 +- .../schemas/total_follower_count.json | 4 +- .../schemas/ugc_posts.json | 293 +-- .../source_linkedin_pages/source.py | 12 + .../unit_tests/test_source.py | 5 +- airbyte_github_hg_social_repo_url | 5 + codecov 2.yml | 18 + docs/integrations/README.md | 1 + docs/integrations/sources/linkedin-pages.md | 112 ++ 19 files changed, 257 insertions(+), 2455 deletions(-) create mode 100644 CODE_OF_CONDUCT 2.md create mode 100644 CONTRIBUTING 2.md create mode 100644 airbyte_github_hg_social_repo_url create mode 100644 codecov 2.yml create mode 100644 docs/integrations/sources/linkedin-pages.md diff --git a/.vscode/settings.json b/.vscode/settings.json index f033cb881e892..3c39f6028ac33 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -31,6 +31,9 @@ }, "[json]": { "editor.formatOnSave": true, - "editor.defaultFormatter": "esbenp.prettier-vscode" - } + "editor.defaultFormatter": "vscode.json-language-features" + }, + "python.analysis.extraPaths": [ + "./airbyte-cdk/python" + ] } diff --git a/CODE_OF_CONDUCT 2.md b/CODE_OF_CONDUCT 2.md new file mode 100644 index 0000000000000..7d0ec94e63928 --- /dev/null +++ b/CODE_OF_CONDUCT 2.md @@ -0,0 +1,2 @@ +# Code of conduct +View in [docs.airbyte.io](https://docs.airbyte.io/contributing-to-airbyte/code-of-conduct) diff --git a/CONTRIBUTING 2.md b/CONTRIBUTING 2.md new file mode 100644 index 0000000000000..85512b1d4afa6 --- /dev/null +++ b/CONTRIBUTING 2.md @@ -0,0 +1,2 @@ +# Contributing +View on [docs.airbyte.io](https://docs.airbyte.io/contributing-to-airbyte) diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 3de730a7634a8..6d57ebd743155 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -52,6 +52,7 @@ | Iterable | [![source-iterable](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-iterable%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-iterable) | | Jira | [![source-jira](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-jira%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-jira) | | LinkedIn Ads | [![source-linkedin-ads](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-linkedin-ads%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-linkedin-ads) | +| LinkedIn Pages | | | Linnworks | [![source-linnworks](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-linnworks%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-linnworks) | | Lever Hiring | [![source-lever-hiring](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-lever-hiring%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-lever-hiring) | | Looker | [![source-looker](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-looker%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-looker) | diff --git a/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile b/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile index 6eff6390b8787..926f0c238fb0a 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile +++ b/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7.11-alpine3.14 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder diff --git a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml index 15191f1b4d4d6..cbdcf258c7f03 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml @@ -1,6 +1,6 @@ # See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests -connector_image: airbyte/source-linkedin-pages:dev0.1.1 +connector_image: airbyte/source-linkedin-pages:dev tests: spec: - spec_path: "source_linkedin_pages/spec.json" @@ -14,17 +14,6 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes - # incremental: # TODO if your connector does not implement incremental sync, remove this block - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + configured_catalog_path: "integration_tests/configured_catalog.json" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json index 1e1d0298f3d93..248f4dcca47f5 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -1,201 +1,24 @@ { - "type": "object", + "type": ["null", "object"], "properties": { "paging": { - "type": "object", + "type": ["null", "object"], "properties": { "start": { - "type": "integer" + "type": ["null", "integer"] }, "count": { - "type": "integer" + "type": ["null", "integer"] }, "links": { - "type": "array", + "type": ["null", "array"], "items": {} } } }, "elements": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "followerCountsByAssociationType": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "followerCounts": { - "type": "object", - "properties": { - "organicFollowerCount": { - "type": "integer" - }, - "paidFollowerCount": { - "type": "integer" - } - } - }, - "associationType": { - "type": "string" - } - } - } - ] - }, - "followerCountsByRegion": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "region": { - "type": "string" - }, - "followerCounts": { - "type": "object", - "properties": { - "organicFollowerCount": { - "type": "integer" - }, - "paidFollowerCount": { - "type": "integer" - } - } - } - } - } - ] - }, - "followerCountsBySeniority": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "followerCounts": { - "type": "object", - "properties": { - "organicFollowerCount": { - "type": "integer" - }, - "paidFollowerCount": { - "type": "integer" - } - } - }, - "seniority": { - "type": "string" - } - } - } - ] - }, - "followerCountsByIndustry": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "followerCounts": { - "type": "object", - "properties": { - "organicFollowerCount": { - "type": "integer" - }, - "paidFollowerCount": { - "type": "integer" - } - } - }, - "industry": { - "type": "string" - } - } - } - ] - }, - "followerCountsByFunction": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "followerCounts": { - "type": "object", - "properties": { - "organicFollowerCount": { - "type": "integer" - }, - "paidFollowerCount": { - "type": "integer" - } - } - }, - "function": { - "type": "string" - } - } - } - ] - }, - "followerCountsByStaffCountRange": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "followerCounts": { - "type": "object", - "properties": { - "organicFollowerCount": { - "type": "integer" - }, - "paidFollowerCount": { - "type": "integer" - } - } - }, - "staffCountRange": { - "type": "string" - } - } - } - ] - }, - "followerCountsByCountry": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "followerCounts": { - "type": "object", - "properties": { - "organicFollowerCount": { - "type": "integer" - }, - "paidFollowerCount": { - "type": "integer" - } - } - }, - "country": { - "type": "string" - } - } - } - ] - }, - "organizationalEntity": { - "type": "string" - } - } - } - ] + "type": ["null", "array"], + "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json index 11e1446820854..badbb4746301e 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -1,268 +1,202 @@ { - "type": "object", + "type": ["null", "object"], "properties": { "vanityName": { - "type": "string" + "type": ["null", "string"] }, "localizedName": { - "type": "string" + "type": ["null", "string"] }, "website": { - "type": "object", + "type": ["null", "object"], "properties": { "localized": { - "type": "object", + "type": ["null", "object"], "properties": { "en_US": { - "type": "string" + "type": ["null", "string"] } } }, "preferredLocale": { - "type": "object", + "type": ["null", "object"], "properties": { "country": { - "type": "string" + "type": ["null", "string"] }, "language": { - "type": "string" + "type": ["null", "string"] } } } } }, "foundedOn": { - "type": "object", + "type": ["null", "object"], "properties": { "year": { - "type": "integer" + "type": ["null", "integer"] } } }, "groups": { - "type": "array", + "type": ["null", "array"], "items": {} }, "description": { - "type": "object", + "type": ["null", "object"], "properties": { "localized": { - "type": "object", + "type": ["null", "object"], "properties": { "en_US": { - "type": "string" + "type": ["null", "string"] } } }, "preferredLocale": { - "type": "object", + "type": ["null", "object"], "properties": { "country": { - "type": "string" + "type": ["null", "string"] }, "language": { - "type": "string" + "type": ["null", "string"] } } } } }, "versionTag": { - "type": "string" + "type": ["null", "string"] }, "coverPhotoV2": { - "type": "object", + "type": ["null", "object"], "properties": { "cropped": { - "type": "string" + "type": ["null", "string"] }, "original": { - "type": "string" + "type": ["null", "string"] }, "cropInfo": { - "type": "object", + "type": ["null", "object"], "properties": { "x": { - "type": "integer" + "type": ["null", "integer"] }, "width": { - "type": "integer" + "type": ["null", "integer"] }, "y": { - "type": "integer" + "type": ["null", "integer"] }, "height": { - "type": "integer" + "type": ["null", "integer"] } } } } }, "defaultLocale": { - "type": "object", + "type": ["null", "object"], "properties": { "country": { - "type": "string" + "type": ["null", "string"] }, "language": { - "type": "string" + "type": ["null", "string"] } } }, "organizationType": { - "type": "string" + "type": ["null", "string"] }, "alternativeNames": { - "type": "array", + "type": ["null", "array"], "items": {} }, "specialties": { - "type": "array", + "type": ["null", "array"], "items": {} }, "staffCountRange": { - "type": "string" + "type": ["null", "string"] }, "localizedSpecialties": { - "type": "array", + "type": ["null", "array"], "items": {} }, "industries": { - "type": "array", - "items": [ - { - "type": "string" - } - ] + "type": ["null", "array"], + "items": {} }, "name": { - "type": "object", + "type": ["null", "object"], "properties": { "localized": { - "type": "object", + "type": ["null", "object"], "properties": { "en_US": { - "type": "string" + "type": ["null", "string"] } } }, "preferredLocale": { - "type": "object", + "type": ["null", "object"], "properties": { "country": { - "type": "string" + "type": ["null", "string"] }, "language": { - "type": "string" + "type": ["null", "string"] } } } } }, "primaryOrganizationType": { - "type": "string" + "type": ["null", "string"] }, "locations": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "locationType": { - "type": "string" - }, - "description": { - "type": "object", - "properties": { - "localized": { - "type": "object", - "properties": { - "en_US": { - "type": "string" - } - } - }, - "preferredLocale": { - "type": "object", - "properties": { - "country": { - "type": "string" - }, - "language": { - "type": "string" - } - } - } - } - }, - "address": { - "type": "object", - "properties": { - "geographicArea": { - "type": "string" - }, - "country": { - "type": "string" - }, - "city": { - "type": "string" - }, - "line1": { - "type": "string" - }, - "postalCode": { - "type": "string" - } - } - }, - "localizedDescription": { - "type": "string" - }, - "geoLocation": { - "type": "string" - }, - "streetAddressFieldState": { - "type": "string" - } - } - } - ] + "type": ["null", "array"], + "items": {} }, "id": { - "type": "integer" + "type": ["null", "integer"] }, "localizedDescription": { - "type": "string" + "type": ["null", "string"] }, "$URN": { - "type": "string" + "type": ["null", "string"] }, "localizedWebsite": { - "type": "string" + "type": ["null", "string"] }, "logoV2": { - "type": "object", + "type": ["null", "object"], "properties": { "cropped": { - "type": "string" + "type": ["null", "string"] }, "original": { - "type": "string" + "type": ["null", "string"] }, "cropInfo": { - "type": "object", + "type": ["null", "object"], "properties": { "x": { - "type": "integer" + "type": ["null", "integer"] }, "width": { - "type": "integer" + "type": ["null", "integer"] }, "y": { - "type": "integer" + "type": ["null", "integer"] }, "height": { - "type": "integer" + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json index a003e0efc4bc0..248f4dcca47f5 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json @@ -1,1726 +1,24 @@ { - "type": "object", + "type": ["null", "object"], "properties": { "paging": { - "type": "object", + "type": ["null", "object"], "properties": { "start": { - "type": "integer" + "type": ["null", "integer"] }, "count": { - "type": "integer" + "type": ["null", "integer"] }, "links": { - "type": "array", + "type": ["null", "array"], "items": {} } } }, "elements": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "pageStatisticsBySeniority": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "pageStatistics": { - "type": "object", - "properties": { - "views": { - "type": "object", - "properties": { - "mobileProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allDesktopPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "insightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allMobilePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "jobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "productsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "peoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "overviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "lifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "careersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopPeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "aboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobilePeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - } - } - } - } - }, - "seniority": { - "type": "string" - } - } - } - ] - }, - "pageStatisticsByCountry": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "country": { - "type": "string" - }, - "pageStatistics": { - "type": "object", - "properties": { - "views": { - "type": "object", - "properties": { - "mobileProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allDesktopPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "insightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allMobilePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "jobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "productsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "peoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "overviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "lifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "careersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopPeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "aboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobilePeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - } - } - } - } - } - } - } - ] - }, - "pageStatisticsByIndustry": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "pageStatistics": { - "type": "object", - "properties": { - "views": { - "type": "object", - "properties": { - "mobileProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allDesktopPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "insightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allMobilePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "jobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "productsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "peoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "overviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "lifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "careersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopPeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "aboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobilePeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - } - } - } - } - } - } - } - ] - }, - "organization": { - "type": "string" - }, - "totalPageStatistics": { - "type": "object", - "properties": { - "clicks": { - "type": "object", - "properties": { - "mobileCareersPageClicks": { - "type": "object", - "properties": { - "careersPageJobsClicks": { - "type": "integer" - }, - "careersPagePromoLinksClicks": { - "type": "integer" - }, - "careersPageEmployeesClicks": { - "type": "integer" - } - } - }, - "careersPageClicks": { - "type": "object", - "properties": { - "careersPagePromoLinksClicks": { - "type": "integer" - }, - "careersPageBannerPromoClicks": { - "type": "integer" - }, - "careersPageJobsClicks": { - "type": "integer" - }, - "careersPageEmployeesClicks": { - "type": "integer" - } - } - } - } - }, - "views": { - "type": "object", - "properties": { - "mobileProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allDesktopPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "insightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allMobilePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "jobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "productsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "peoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "overviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "lifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "careersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopPeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "aboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobilePeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - } - } - } - } - }, - "pageStatisticsByStaffCountRange": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "staffCountRange": { - "type": "string" - }, - "pageStatistics": { - "type": "object", - "properties": { - "views": { - "type": "object", - "properties": { - "mobileProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allDesktopPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "insightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allMobilePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "jobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "productsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "peoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "overviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "lifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "careersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopPeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "aboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobilePeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - } - } - } - } - } - } - } - ] - }, - "pageStatisticsByRegion": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "region": { - "type": "string" - }, - "pageStatistics": { - "type": "object", - "properties": { - "views": { - "type": "object", - "properties": { - "mobileProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allDesktopPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "insightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allMobilePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "jobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "productsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "peoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "overviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "lifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "careersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopPeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "aboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobilePeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - } - } - } - } - } - } - } - ] - }, - "pageStatisticsByFunction": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "pageStatistics": { - "type": "object", - "properties": { - "views": { - "type": "object", - "properties": { - "mobileProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allDesktopPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "insightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allMobilePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "jobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "productsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopProductsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "peoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "overviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "lifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopOverviewPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "allPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "careersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopJobsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopPeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "aboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopAboutPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobilePeoplePageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopCareersPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "desktopLifeAtPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - }, - "mobileInsightsPageViews": { - "type": "object", - "properties": { - "pageViews": { - "type": "integer" - } - } - } - } - } - } - }, - "function": { - "type": "string" - } - } - } - ] - } - } - } - ] + "type": ["null", "array"], + "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json index dfc9a7e6a7115..2bea295e87bf9 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json @@ -1,4 +1,5 @@ { + "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { "paging": { @@ -21,48 +22,7 @@ }, "elements": { "type": "array", - "items": [ - { - "type": "object", - "properties": { - "totalShareStatistics": { - "type": "object", - "properties": { - "uniqueImpressionsCount": { - "type": "integer" - }, - "shareCount": { - "type": "integer" - }, - "shareMentionsCount": { - "type": "integer" - }, - "engagement": { - "type": "number" - }, - "clickCount": { - "type": "integer" - }, - "likeCount": { - "type": "integer" - }, - "impressionCount": { - "type": "integer" - }, - "commentMentionsCount": { - "type": "integer" - }, - "commentCount": { - "type": "integer" - } - } - }, - "organizationalEntity": { - "type": "string" - } - } - } - ] + "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json index 5e0fd75a811ce..621997bd13b52 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json @@ -1,109 +1,27 @@ { - "type": "object", + "type": ["null", "object"], "properties": { "paging": { - "type": "object", + "type": ["null", "object"], "properties": { "start": { - "type": "integer" + "type": ["null", "integer"] }, "count": { - "type": "integer" + "type": ["null", "integer"] }, "links": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "rel": { - "type": "string" - }, - "href": { - "type": "string" - } - } - } - ] + "type": ["null", "array"], + "items": {} }, "total": { - "type": "integer" + "type": ["null", "integer"] } } }, "elements": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "owner": { - "type": "string" - }, - "activity": { - "type": "string" - }, - "edited": { - "type": "boolean" - }, - "resharedShare": { - "type": "string" - }, - "created": { - "type": "object", - "properties": { - "actor": { - "type": "string" - }, - "time": { - "type": "integer" - } - } - }, - "lastModified": { - "type": "object", - "properties": { - "actor": { - "type": "string" - }, - "time": { - "type": "integer" - } - } - }, - "text": { - "type": "object", - "properties": { - "text": { - "type": "string" - } - } - }, - "id": { - "type": "string" - }, - "originalShare": { - "type": "string" - }, - "distribution": { - "type": "object", - "properties": { - "linkedInDistributionTarget": { - "type": "object", - "properties": { - "visibleToGuest": { - "type": "boolean" - } - } - } - } - } - } - } - ] + "type": ["null", "array"], + "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json index 18f07c4bbb778..d48ce8aa10a17 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json @@ -1,8 +1,8 @@ { - "type": "object", + "type": ["null", "object"], "properties": { "firstDegreeSize": { - "type": "integer" + "type": ["null", "integer"] } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json index f95bb41047b7e..621997bd13b52 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json @@ -1,302 +1,27 @@ { - "type": "object", + "type": ["null", "object"], "properties": { "paging": { - "type": "object", + "type": ["null", "object"], "properties": { "start": { - "type": "integer" + "type": ["null", "integer"] }, "count": { - "type": "integer" + "type": ["null", "integer"] }, "links": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "rel": { - "type": "string" - }, - "href": { - "type": "string" - } - } - } - ] + "type": ["null", "array"], + "items": {} }, "total": { - "type": "integer" + "type": ["null", "integer"] } } }, "elements": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "lifecycleState": { - "type": "string" - }, - "specificContent": { - "type": "object", - "properties": { - "com.linkedin.ugc.ShareContent": { - "type": "object", - "properties": { - "shareCommentary": { - "type": "object", - "properties": { - "inferredLocale": { - "type": "string" - }, - "attributes": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "length": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "value": { - "type": "object", - "properties": { - "com.linkedin.common.MemberAttributedEntity": { - "type": "object", - "properties": { - "member": { - "type": "string" - } - } - } - } - } - } - }, - { - "type": "object", - "properties": { - "length": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "value": { - "type": "object", - "properties": { - "com.linkedin.common.HashtagAttributedEntity": { - "type": "object", - "properties": { - "hashtag": { - "type": "string" - } - } - } - } - } - } - }, - { - "type": "object", - "properties": { - "length": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "value": { - "type": "object", - "properties": { - "com.linkedin.common.HashtagAttributedEntity": { - "type": "object", - "properties": { - "hashtag": { - "type": "string" - } - } - } - } - } - } - }, - { - "type": "object", - "properties": { - "length": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "value": { - "type": "object", - "properties": { - "com.linkedin.common.HashtagAttributedEntity": { - "type": "object", - "properties": { - "hashtag": { - "type": "string" - } - } - } - } - } - } - }, - { - "type": "object", - "properties": { - "length": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "value": { - "type": "object", - "properties": { - "com.linkedin.common.HashtagAttributedEntity": { - "type": "object", - "properties": { - "hashtag": { - "type": "string" - } - } - } - } - } - } - } - ] - }, - "text": { - "type": "string" - } - } - }, - "media": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "thumbnails": { - "type": "array", - "items": {} - }, - "status": { - "type": "string" - }, - "media": { - "type": "string" - } - } - } - ] - }, - "shareFeatures": { - "type": "object", - "properties": { - "hashtags": { - "type": "array", - "items": [ - { - "type": "string" - }, - { - "type": "string" - }, - { - "type": "string" - }, - { - "type": "string" - } - ] - } - } - }, - "shareMediaCategory": { - "type": "string" - } - } - } - } - }, - "visibility": { - "type": "object", - "properties": { - "com.linkedin.ugc.MemberNetworkVisibility": { - "type": "string" - } - } - }, - "created": { - "type": "object", - "properties": { - "actor": { - "type": "string" - }, - "time": { - "type": "integer" - } - } - }, - "author": { - "type": "string" - }, - "versionTag": { - "type": "string" - }, - "id": { - "type": "string" - }, - "firstPublishedAt": { - "type": "integer" - }, - "lastModified": { - "type": "object", - "properties": { - "actor": { - "type": "string" - }, - "time": { - "type": "integer" - } - } - }, - "distribution": { - "type": "object", - "properties": { - "externalDistributionChannels": { - "type": "array", - "items": {} - }, - "distributedViaFollowFeed": { - "type": "boolean" - }, - "feedDistribution": { - "type": "string" - } - } - }, - "contentCertificationRecord": { - "type": "string" - } - } - } - ] + "type": ["null", "array"], + "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py index 214923cc0f94c..2dfa602c784c7 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py @@ -45,6 +45,18 @@ def parse_response( ) -> Iterable[Mapping]: return [response.json()] + def should_retry(self, response: requests.Response) -> bool: + if response.status_code == 429: + error_message = ( + f"Stream {self.name}: LinkedIn API requests are rate limited. " + f"Rate limits specify the maximum number of API calls that can be made in a 24 hour period. " + f"These limits reset at midnight UTC every day. " + f"You can find more information here https://docs.airbyte.io/integrations/sources/linkedin-ads. " + f"Also quotas and usage are here: https://www.linkedin.com/developers/apps." + ) + self.logger.error(error_message) + return super().should_retry(response) + class OrganizationLookup(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: diff --git a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py index 7f23f89ed1806..5609fea61d95a 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py @@ -17,6 +17,5 @@ def test_streams(mocker): source = SourceLinkedinPages() config_mock = MagicMock() streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 2 - assert len(streams) == expected_streams_number + expected_streams_number = 7 + assert len(streams) == expected_streams_number \ No newline at end of file diff --git a/airbyte_github_hg_social_repo_url b/airbyte_github_hg_social_repo_url new file mode 100644 index 0000000000000..6fce7b201a911 --- /dev/null +++ b/airbyte_github_hg_social_repo_url @@ -0,0 +1,5 @@ +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 + +https://airbyteuser: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@github.com/Hunt-Gather-Create/hg_social_media_analytics.git + +https://github.com/Hunt-Gather-Create/hg_social_media_analytics.git \ No newline at end of file diff --git a/codecov 2.yml b/codecov 2.yml new file mode 100644 index 0000000000000..55a0f1f89c3c0 --- /dev/null +++ b/codecov 2.yml @@ -0,0 +1,18 @@ +codecov: + notify: + require_ci_to_pass: no + +coverage: + status: + patch: + default: + target: 90% + if_no_uploads: error + if_not_found: failure + if_ci_failed: failure + project: + default: + target: 90% + if_no_uploads: error + if_not_found: failure + if_ci_failed: failure diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 27a819813d78d..9aee5c9f310ee 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -88,6 +88,7 @@ For more information about the grading system, see [Product Release Stages](http | [Lemlist](sources/lemlist.md) | Alpha | Yes | | [Lever](sources/lever-hiring.md) | Alpha | No | | [LinkedIn Ads](sources/linkedin-ads.md) | Generally Available | Yes | +| [LinkedIn Pages](sources/linkedin-pages.md) | Alpha | Yes | | [Linnworks](sources/linnworks.md) | Alpha | Yes | | [Looker](sources/looker.md) | Alpha | Yes | | [Magento](sources/magento.md) | Alpha | No | diff --git a/docs/integrations/sources/linkedin-pages.md b/docs/integrations/sources/linkedin-pages.md new file mode 100644 index 0000000000000..743604a7fa14b --- /dev/null +++ b/docs/integrations/sources/linkedin-pages.md @@ -0,0 +1,112 @@ +# LinkedIn Pages + +## Sync overview + +The LinkedIn Pages source only supports Full Refresh for now. Incremental Sync will be coming soon. + +This Source Connector is based on a [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). Airbyte uses [LinkedIn Marketing Developer Platform - API](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/marketing-integrations-overview) to fetch data from LinkedIn Pages. + +### Output schema + +This Source is capable of syncing the following data as streams: + +* [Organization Lookup](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/organization-lookup-api?tabs=http#retrieve-organizations) +* [Follower Statistics](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/follower-statistics?tabs=http#retrieve-lifetime-follower-statistics) +* [Page Statistics](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/page-statistics?tabs=http#retrieve-lifetime-organization-page-statistics) +* [Share Statistics](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/share-statistics?tabs=http#retrieve-lifetime-share-statistics) +* [Shares (Latest 50)](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/shares/share-api?tabs=http#find-shares-by-owner) +* [Total Follower Count](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/organization-lookup-api?tabs=http#retrieve-organization-follower-count) +* [UGC Posts](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/shares/ugc-post-api?tabs=http#find-ugc-posts-by-authors) + +### NOTE: + +All streams only sync all-time statistics at this time. A `start_date` field will be added soon to pull data starting at a single point in time. + +### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--------------- | :----------- | :------------------------- | +| `number` | `number` | float number | +| `integer` | `integer` | whole number | +| `array` | `array` | | +| `boolean` | `boolean` | True/False | +| `string` | `string` | | + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------------------------------- | :------------------- | :---- | +| Full Refresh Overwrite Sync | Yes | | +| Full Refresh Append Sync | No | | +| Incremental - Append Sync | No | | +| Incremental - Append + Deduplication Sync | No | | +| Namespaces | No | | + +### Performance considerations + +There are official Rate Limits for LinkedIn Pages API Usage, [more information here](https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/rate-limits?context=linkedin/marketing/context). Rate limited requests will receive a 429 response. Rate limits specify the maximum number of API calls that can be made in a 24 hour period. These limits reset at midnight UTC every day. In rare cases, LinkedIn may also return a 429 response as part of infrastructure protection. API service will return to normal automatically. In such cases you will receive the next error message: + +```text +"Caught retryable error ' or null' after tries. Waiting seconds then retrying..." +``` + +This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Error. If the maximum of available API requests capacity is reached, you will have the following message: + +```text +"Max try rate limit exceded..." +``` + +After 5 unsuccessful attempts - the connector will stop the sync operation. In such cases check your Rate Limits [on this page](https://www.linkedin.com/developers/apps) > Choose your app > Analytics. + +## Getting started +The API user account should be assigned the following permissions for the API endpoints: +Endpoints such as: `Organization Lookup API`, `Follower Statistics`, `Page Statistics`, `Share Statistics`, `Shares`, `UGC Posts` require these permissions: +* `r_organization_social`: Retrieve your organization's posts, comments, reactions, and other engagement data. +* `rw_organization_admin`: Manage your organization's pages and retrieve reporting data. + +The API user account should be assigned the `ADMIN` role. + +### Authentication +There are 2 authentication methods: Access Token or OAuth2.0. +OAuth2.0 is recommended since it will continue streaming data for 12 months instead of 2 months with an access token. + +##### Create the `Refresh_Token` or `Access_Token`: +The source LinkedIn Pages can use either the `client_id`, `client_secret` and `refresh_token` for OAuth2.0 authentication or simply use an `access_token` in the UI connector's settings to make API requests. Access tokens expire after `2 months from creation date (60 days)` and require a user to manually authenticate again. Refresh tokens expire after `12 months from creation date (365 days)`. If you receive a `401 invalid token response`, the error logs will state that your token has expired and to re-authenticate your connection to generate a new token. This is described more [here](https://docs.microsoft.com/en-us/linkedin/shared/authentication/authorization-code-flow?context=linkedin/context). + +1. **Log in to LinkedIn as the API user** + +2. **Create an App** [here](https://www.linkedin.com/developers/apps): + * `App Name`: airbyte-source + * `Company`: search and find your LinkedIn Company Page + * `Privacy policy URL`: link to company privacy policy + * `Business email`: developer/admin email address + * `App logo`: Airbyte's \(or Company's\) logo + * Review/agree to legal terms and create app + * Review the **Auth** tab: + * **Save your `client_id` and `client_secret`** \(for later steps\) + * Oauth 2.0 settings: Provide a `redirect_uri` \(for later steps\): `https://airbyte.io` + +3. **Verify App**: + * In the **Settings** tab of your app dashboard, you'll see a **Verify** button. Click that button! + * Generate and provide the verify URL to your Company's LinkedIn Admin to verify the app. + +4. **Request API Access**: + * Navigate to the **Products** tab + * Select the [Marketing Developer Platform](https://docs.microsoft.com/en-us/linkedin/marketing/) and agree to the legal terms + * After a few minutes, refresh the page to see a link to `View access form` in place of the **Select** button + * Fill out the access form and access should be granted **within 72 hours** (usually quicker) + +5. **Create A Refresh Token** (or Access Token): + * Navigate to the LinkedIn Developers' [OAuth Token Tools](https://www.linkedin.com/developers/tools/oauth) and click **Create token** + * Select your newly created app and check the boxes for the following scopes: + * `r_organization_social` + * `rw_organization_admin` + * Click **Request access token** and once generated, **save your Refresh token** + +6. **Use the `client_id`, `client_secret` and `refresh_token`** from Steps 2 and 5 to autorize the LinkedIn Pages connector within the Airbyte UI. + * As mentioned earlier, you can also simply use the Access token auth method for 60-day access. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------- | From 22250adddabf5fcd57c04206b8d4c732aad82cf9 Mon Sep 17 00:00:00 2001 From: Jordan Scott Date: Tue, 31 May 2022 16:08:54 -0500 Subject: [PATCH 09/21] requested changes completed --- .vscode/settings.json | 9 +- CODE_OF_CONDUCT 2.md | 2 - CONTRIBUTING 2.md | 2 - .../acceptance-test-config.yml | 2 +- .../integration_tests/abnormal_state.json | 2 +- .../integration_tests/sample_config.json | 2 +- .../schemas/follower_statistics.json | 87 +++- .../schemas/organization_lookup.json | 416 +++++++++++++++--- .../source_linkedin_pages/source.py | 25 +- codecov 2.yml | 18 - 10 files changed, 442 insertions(+), 123 deletions(-) delete mode 100644 CODE_OF_CONDUCT 2.md delete mode 100644 CONTRIBUTING 2.md delete mode 100644 codecov 2.yml diff --git a/.vscode/settings.json b/.vscode/settings.json index 3c39f6028ac33..ebb6af501317b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -31,9 +31,6 @@ }, "[json]": { "editor.formatOnSave": true, - "editor.defaultFormatter": "vscode.json-language-features" - }, - "python.analysis.extraPaths": [ - "./airbyte-cdk/python" - ] -} + "editor.defaultFormatter": "esbenp.prettier-vscode" + } +} \ No newline at end of file diff --git a/CODE_OF_CONDUCT 2.md b/CODE_OF_CONDUCT 2.md deleted file mode 100644 index 7d0ec94e63928..0000000000000 --- a/CODE_OF_CONDUCT 2.md +++ /dev/null @@ -1,2 +0,0 @@ -# Code of conduct -View in [docs.airbyte.io](https://docs.airbyte.io/contributing-to-airbyte/code-of-conduct) diff --git a/CONTRIBUTING 2.md b/CONTRIBUTING 2.md deleted file mode 100644 index 85512b1d4afa6..0000000000000 --- a/CONTRIBUTING 2.md +++ /dev/null @@ -1,2 +0,0 @@ -# Contributing -View on [docs.airbyte.io](https://docs.airbyte.io/contributing-to-airbyte) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml index cbdcf258c7f03..b7023bc46b224 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml @@ -16,4 +16,4 @@ tests: configured_catalog_path: "integration_tests/configured_catalog.json" full_refresh: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" \ No newline at end of file + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json index ceae7331d2d9a..8f9f142bd9120 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json @@ -20,4 +20,4 @@ "ugc_posts": { "end_date": "2050-01-01" } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json index c06b53f2c96e3..fd416acb2613e 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json @@ -2,6 +2,6 @@ "org_id": 12345678, "credentials": { "auth_method": "access_token", - "access_token": "drAL4JXBJ6v1qaU9iWargosra6ibiw0ZWEdMnC0ZizeD1gLRQP6u1pkQ" + "access_token": "example_token_string123" } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json index 248f4dcca47f5..8d06eb23cb2ce 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -1,24 +1,77 @@ { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "paging": { - "type": ["null", "object"], - "properties": { - "start": { - "type": ["null", "integer"] - }, - "count": { - "type": ["null", "integer"] - }, - "links": { - "type": ["null", "array"], - "items": {} + "elements": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "followerCountsByAssociationType": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByRegion": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsBySeniority": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByIndustry": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByFunction": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByStaffCountRange": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByCountry": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "organizationalEntity": { + "type": [ + "null", + "string" + ] + } } } - }, - "elements": { - "type": ["null", "array"], - "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json index badbb4746301e..82ce72ca251f4 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -1,202 +1,494 @@ { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "vanityName": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "localizedName": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "website": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "language": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } } }, "foundedOn": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "year": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] } } }, "groups": { - "type": ["null", "array"], - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "description": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "language": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } } }, "versionTag": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "coverPhotoV2": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "cropped": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "original": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "cropInfo": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "x": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "width": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "y": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "height": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] } } } } }, "defaultLocale": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "language": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "organizationType": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "alternativeNames": { - "type": ["null", "array"], - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "specialties": { - "type": ["null", "array"], - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "staffCountRange": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "localizedSpecialties": { - "type": ["null", "array"], - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "industries": { - "type": ["null", "array"], - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } }, "name": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "language": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } } }, "primaryOrganizationType": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "locations": { - "type": ["null", "array"], - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "locationType": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "object" + ], + "properties": { + "localized": { + "type": [ + "null", + "object" + ], + "properties": { + "en_US": { + "type": [ + "null", + "string" + ] + } + } + }, + "preferredLocale": { + "type": [ + "null", + "object" + ], + "properties": { + "country": { + "type": [ + "null", + "string" + ] + }, + "language": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "address": { + "type": [ + "null", + "object" + ], + "properties": { + "geographicArea": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "line1": { + "type": [ + "null", + "string" + ] + }, + "postalCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "localizedDescription": { + "type": [ + "null", + "string" + ] + }, + "geoLocation": { + "type": [ + "null", + "string" + ] + }, + "streetAddressFieldState": { + "type": [ + "null", + "string" + ] + } + } + } }, "id": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "localizedDescription": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "$URN": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "localizedWebsite": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "logoV2": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "cropped": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "original": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "cropInfo": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "x": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "width": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "y": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "height": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] } } } diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py index 2dfa602c784c7..a8c80c5efc82c 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py @@ -23,13 +23,13 @@ class LinkedinPagesStream(HttpStream, ABC): def __init__(self, config): super().__init__(authenticator=config.get("authenticator")) self.config = config - + @property def org(self): """Property to return the user Organization Id from input""" return self.config.get("org_id") - + def path(self, **kwargs) -> str: """Returns the API endpoint path for stream, from `endpoint` class attribute.""" return self.endpoint @@ -51,7 +51,7 @@ def should_retry(self, response: requests.Response) -> bool: f"Stream {self.name}: LinkedIn API requests are rate limited. " f"Rate limits specify the maximum number of API calls that can be made in a 24 hour period. " f"These limits reset at midnight UTC every day. " - f"You can find more information here https://docs.airbyte.io/integrations/sources/linkedin-ads. " + f"You can find more information here https://docs.airbyte.io/integrations/sources/linkedin-pages. " f"Also quotas and usage are here: https://www.linkedin.com/developers/apps." ) self.logger.error(error_message) @@ -60,49 +60,49 @@ def should_retry(self, response: requests.Response) -> bool: class OrganizationLookup(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + path = f"organizations/{self.org}" return path class FollowerStatistics(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + path = f"organizationalEntityFollowerStatistics?q=organizationalEntity&organizationalEntity=urn:li:organization:{self.org}" return path class PageStatistics(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + path = f"organizationPageStatistics?q=organization&organization=urn%3Ali%3Aorganization%3A{self.org}" return path class ShareStatistics(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + path = f"organizationalEntityShareStatistics?q=organizationalEntity&organizationalEntity=urn%3Ali%3Aorganization%3A{self.org}" return path class Shares(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + path = f"shares?q=owners&owners=urn%3Ali%3Aorganization%3A{self.org}&sortBy=LAST_MODIFIED&sharesPerOwner=50" return path class TotalFollowerCount(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + path = f"networkSizes/urn:li:organization:{self.org}?edgeType=CompanyFollowedByMember" return path - + class UgcPosts(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + path = f"ugcPosts?q=authors&authors=List(urn%3Ali%3Aorganization%3A{self.org})&sortBy=LAST_MODIFIED&count=50" return path @@ -161,7 +161,7 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> return True, None except Exception as e: return False, e - + # RUN: $ python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json def streams(self, config: Mapping[str, Any]) -> List[Stream]: @@ -175,4 +175,3 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: TotalFollowerCount(config), UgcPosts(config) ] - diff --git a/codecov 2.yml b/codecov 2.yml deleted file mode 100644 index 55a0f1f89c3c0..0000000000000 --- a/codecov 2.yml +++ /dev/null @@ -1,18 +0,0 @@ -codecov: - notify: - require_ci_to_pass: no - -coverage: - status: - patch: - default: - target: 90% - if_no_uploads: error - if_not_found: failure - if_ci_failed: failure - project: - default: - target: 90% - if_no_uploads: error - if_not_found: failure - if_ci_failed: failure From 58dba365fbd1509308eea3d26e47fb3d8868539b Mon Sep 17 00:00:00 2001 From: Jordan Scott Date: Mon, 23 May 2022 13:46:52 -0500 Subject: [PATCH 10/21] fixed last source acceptance test issue and passed all --- .vscode/settings.json | 2 +- .../schemas/follower_statistics.json | 87 +--- .../schemas/organization_lookup.json | 416 +++--------------- 3 files changed, 80 insertions(+), 425 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index ebb6af501317b..f033cb881e892 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -33,4 +33,4 @@ "editor.formatOnSave": true, "editor.defaultFormatter": "esbenp.prettier-vscode" } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json index 8d06eb23cb2ce..248f4dcca47f5 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -1,77 +1,24 @@ { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { - "elements": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "object" - ], - "properties": { - "followerCountsByAssociationType": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByRegion": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsBySeniority": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByIndustry": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByFunction": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByStaffCountRange": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByCountry": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "organizationalEntity": { - "type": [ - "null", - "string" - ] - } + "paging": { + "type": ["null", "object"], + "properties": { + "start": { + "type": ["null", "integer"] + }, + "count": { + "type": ["null", "integer"] + }, + "links": { + "type": ["null", "array"], + "items": {} } } + }, + "elements": { + "type": ["null", "array"], + "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json index 82ce72ca251f4..ec853967758b9 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -1,494 +1,202 @@ { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "vanityName": { - "type": [ - "null", - "string" - ] + "type": "string" }, "localizedName": { - "type": [ - "null", - "string" - ] + "type": "string" }, "website": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "localized": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "en_US": { - "type": [ - "null", - "string" - ] + "type": "string" } } }, "preferredLocale": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "country": { - "type": [ - "null", - "string" - ] + "type": "string" }, "language": { - "type": [ - "null", - "string" - ] + "type": "string" } } } } }, "foundedOn": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "year": { - "type": [ - "null", - "integer" - ] + "type": "integer" } } }, "groups": { - "type": [ - "null", - "array" - ], - "items": { - "items": {} - } + "type": "array", + "items": {} }, "description": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "localized": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "en_US": { - "type": [ - "null", - "string" - ] + "type": "string" } } }, "preferredLocale": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "country": { - "type": [ - "null", - "string" - ] + "type": "string" }, "language": { - "type": [ - "null", - "string" - ] + "type": "string" } } } } }, "versionTag": { - "type": [ - "null", - "string" - ] + "type": "string" }, "coverPhotoV2": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "cropped": { - "type": [ - "null", - "string" - ] + "type": "string" }, "original": { - "type": [ - "null", - "string" - ] + "type": "string" }, "cropInfo": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "x": { - "type": [ - "null", - "integer" - ] + "type": "integer" }, "width": { - "type": [ - "null", - "integer" - ] + "type": "integer" }, "y": { - "type": [ - "null", - "integer" - ] + "type": "integer" }, "height": { - "type": [ - "null", - "integer" - ] + "type": "integer" } } } } }, "defaultLocale": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "country": { - "type": [ - "null", - "string" - ] + "type": "string" }, "language": { - "type": [ - "null", - "string" - ] + "type": "string" } } }, "organizationType": { - "type": [ - "null", - "string" - ] + "type": "string" }, "alternativeNames": { - "type": [ - "null", - "array" - ], - "items": { - "items": {} - } + "type": "array", + "items": {} }, "specialties": { - "type": [ - "null", - "array" - ], - "items": { - "items": {} - } + "type": "array", + "items": {} }, "staffCountRange": { - "type": [ - "null", - "string" - ] + "type": "string" }, "localizedSpecialties": { - "type": [ - "null", - "array" - ], - "items": { - "items": {} - } + "type": "array", + "items": {} }, "industries": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "string" - ] - } + "type": "array", + "items": {} }, "name": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "localized": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "en_US": { - "type": [ - "null", - "string" - ] + "type": "string" } } }, "preferredLocale": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "country": { - "type": [ - "null", - "string" - ] + "type": "string" }, "language": { - "type": [ - "null", - "string" - ] + "type": "string" } } } } }, "primaryOrganizationType": { - "type": [ - "null", - "string" - ] + "type": "string" }, "locations": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "object" - ], - "properties": { - "locationType": { - "type": [ - "null", - "string" - ] - }, - "description": { - "type": [ - "null", - "object" - ], - "properties": { - "localized": { - "type": [ - "null", - "object" - ], - "properties": { - "en_US": { - "type": [ - "null", - "string" - ] - } - } - }, - "preferredLocale": { - "type": [ - "null", - "object" - ], - "properties": { - "country": { - "type": [ - "null", - "string" - ] - }, - "language": { - "type": [ - "null", - "string" - ] - } - } - } - } - }, - "address": { - "type": [ - "null", - "object" - ], - "properties": { - "geographicArea": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "city": { - "type": [ - "null", - "string" - ] - }, - "line1": { - "type": [ - "null", - "string" - ] - }, - "postalCode": { - "type": [ - "null", - "string" - ] - } - } - }, - "localizedDescription": { - "type": [ - "null", - "string" - ] - }, - "geoLocation": { - "type": [ - "null", - "string" - ] - }, - "streetAddressFieldState": { - "type": [ - "null", - "string" - ] - } - } - } + "type": "array", + "items": {} }, "id": { - "type": [ - "null", - "integer" - ] + "type": "integer" }, "localizedDescription": { - "type": [ - "null", - "string" - ] + "type": "string" }, "$URN": { - "type": [ - "null", - "string" - ] + "type": "string" }, "localizedWebsite": { - "type": [ - "null", - "string" - ] + "type": "string" }, "logoV2": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "cropped": { - "type": [ - "null", - "string" - ] + "type": "string" }, "original": { - "type": [ - "null", - "string" - ] + "type": "string" }, "cropInfo": { - "type": [ - "null", - "object" - ], + "type": "object", "properties": { "x": { - "type": [ - "null", - "integer" - ] + "type": "integer" }, "width": { - "type": [ - "null", - "integer" - ] + "type": "integer" }, "y": { - "type": [ - "null", - "integer" - ] + "type": "integer" }, "height": { - "type": [ - "null", - "integer" - ] + "type": "integer" } } } From 16a97e9c4b37c13516d080f9b3a87bbdb410bbe7 Mon Sep 17 00:00:00 2001 From: Jordan Scott Date: Tue, 31 May 2022 16:08:54 -0500 Subject: [PATCH 11/21] requested changes completed --- .../schemas/follower_statistics.json | 87 +++- .../schemas/organization_lookup.json | 416 +++++++++++++++--- 2 files changed, 424 insertions(+), 79 deletions(-) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json index 248f4dcca47f5..8d06eb23cb2ce 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -1,24 +1,77 @@ { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "paging": { - "type": ["null", "object"], - "properties": { - "start": { - "type": ["null", "integer"] - }, - "count": { - "type": ["null", "integer"] - }, - "links": { - "type": ["null", "array"], - "items": {} + "elements": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "followerCountsByAssociationType": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByRegion": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsBySeniority": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByIndustry": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByFunction": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByStaffCountRange": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "followerCountsByCountry": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "organizationalEntity": { + "type": [ + "null", + "string" + ] + } } } - }, - "elements": { - "type": ["null", "array"], - "items": {} } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json index ec853967758b9..82ce72ca251f4 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -1,202 +1,494 @@ { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "vanityName": { - "type": "string" + "type": [ + "null", + "string" + ] }, "localizedName": { - "type": "string" + "type": [ + "null", + "string" + ] }, "website": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": "string" + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": "string" + "type": [ + "null", + "string" + ] }, "language": { - "type": "string" + "type": [ + "null", + "string" + ] } } } } }, "foundedOn": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "year": { - "type": "integer" + "type": [ + "null", + "integer" + ] } } }, "groups": { - "type": "array", - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "description": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": "string" + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": "string" + "type": [ + "null", + "string" + ] }, "language": { - "type": "string" + "type": [ + "null", + "string" + ] } } } } }, "versionTag": { - "type": "string" + "type": [ + "null", + "string" + ] }, "coverPhotoV2": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "cropped": { - "type": "string" + "type": [ + "null", + "string" + ] }, "original": { - "type": "string" + "type": [ + "null", + "string" + ] }, "cropInfo": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "x": { - "type": "integer" + "type": [ + "null", + "integer" + ] }, "width": { - "type": "integer" + "type": [ + "null", + "integer" + ] }, "y": { - "type": "integer" + "type": [ + "null", + "integer" + ] }, "height": { - "type": "integer" + "type": [ + "null", + "integer" + ] } } } } }, "defaultLocale": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": "string" + "type": [ + "null", + "string" + ] }, "language": { - "type": "string" + "type": [ + "null", + "string" + ] } } }, "organizationType": { - "type": "string" + "type": [ + "null", + "string" + ] }, "alternativeNames": { - "type": "array", - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "specialties": { - "type": "array", - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "staffCountRange": { - "type": "string" + "type": [ + "null", + "string" + ] }, "localizedSpecialties": { - "type": "array", - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } }, "industries": { - "type": "array", - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } }, "name": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": "string" + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": "string" + "type": [ + "null", + "string" + ] }, "language": { - "type": "string" + "type": [ + "null", + "string" + ] } } } } }, "primaryOrganizationType": { - "type": "string" + "type": [ + "null", + "string" + ] }, "locations": { - "type": "array", - "items": {} + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "locationType": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "object" + ], + "properties": { + "localized": { + "type": [ + "null", + "object" + ], + "properties": { + "en_US": { + "type": [ + "null", + "string" + ] + } + } + }, + "preferredLocale": { + "type": [ + "null", + "object" + ], + "properties": { + "country": { + "type": [ + "null", + "string" + ] + }, + "language": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "address": { + "type": [ + "null", + "object" + ], + "properties": { + "geographicArea": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "line1": { + "type": [ + "null", + "string" + ] + }, + "postalCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "localizedDescription": { + "type": [ + "null", + "string" + ] + }, + "geoLocation": { + "type": [ + "null", + "string" + ] + }, + "streetAddressFieldState": { + "type": [ + "null", + "string" + ] + } + } + } }, "id": { - "type": "integer" + "type": [ + "null", + "integer" + ] }, "localizedDescription": { - "type": "string" + "type": [ + "null", + "string" + ] }, "$URN": { - "type": "string" + "type": [ + "null", + "string" + ] }, "localizedWebsite": { - "type": "string" + "type": [ + "null", + "string" + ] }, "logoV2": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "cropped": { - "type": "string" + "type": [ + "null", + "string" + ] }, "original": { - "type": "string" + "type": [ + "null", + "string" + ] }, "cropInfo": { - "type": "object", + "type": [ + "null", + "object" + ], "properties": { "x": { - "type": "integer" + "type": [ + "null", + "integer" + ] }, "width": { - "type": "integer" + "type": [ + "null", + "integer" + ] }, "y": { - "type": "integer" + "type": [ + "null", + "integer" + ] }, "height": { - "type": "integer" + "type": [ + "null", + "integer" + ] } } } From 79f39c23bf989587df9eb6eb87a47db5d59a568c Mon Sep 17 00:00:00 2001 From: Jordan Scott Date: Wed, 1 Jun 2022 13:44:11 -0500 Subject: [PATCH 12/21] added nulls to share_statistics schema --- .../schemas/share_statistics.json | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json index 2bea295e87bf9..621997bd13b52 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json @@ -1,27 +1,26 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", + "type": ["null", "object"], "properties": { "paging": { - "type": "object", + "type": ["null", "object"], "properties": { "start": { - "type": "integer" + "type": ["null", "integer"] }, "count": { - "type": "integer" + "type": ["null", "integer"] }, "links": { - "type": "array", + "type": ["null", "array"], "items": {} }, "total": { - "type": "integer" + "type": ["null", "integer"] } } }, "elements": { - "type": "array", + "type": ["null", "array"], "items": {} } } From 98d907a7aa48600375e822d6fac1ddb688f35876 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 11 Aug 2022 09:43:14 -0300 Subject: [PATCH 13/21] remove utg posts, page stats and other stream --- .../integration_tests/acceptance.py | 4 +- .../integration_tests/configured_catalog.json | 27 ------ .../connectors/source-linkedin-pages/main.py | 4 +- .../connectors/source-linkedin-pages/setup.py | 2 +- .../source_linkedin_pages/analytics.py | 4 +- .../schemas/follower_statistics.json | 94 +++++-------------- .../schemas/organization_lookup.json | 7 +- .../schemas/page_statistics.json | 24 ----- .../schemas/share_statistics.json | 37 +++++--- .../source_linkedin_pages/schemas/shares.json | 27 ------ .../schemas/total_follower_count.json | 4 +- .../schemas/ugc_posts.json | 27 ------ .../source_linkedin_pages/source.py | 51 +++------- .../source_linkedin_pages/utils.py | 4 +- .../unit_tests/test_incremental_streams.py | 59 ------------ .../unit_tests/test_source.py | 21 ----- .../unit_tests/test_streams.py | 83 ---------------- airbyte_github_hg_social_repo_url | 5 - 18 files changed, 77 insertions(+), 407 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json delete mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json delete mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json delete mode 100644 airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_incremental_streams.py delete mode 100644 airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py delete mode 100644 airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_streams.py delete mode 100644 airbyte_github_hg_social_repo_url diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py index 1b85ccdcde521..1d66fbf1a331e 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -10,4 +10,4 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - yield \ No newline at end of file + yield diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json index 6d5c31ddc7549..c0e203e604793 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json @@ -18,15 +18,6 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, - { - "stream": { - "name": "page_statistics", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, { "stream": { "name": "share_statistics", @@ -36,15 +27,6 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, - { - "stream": { - "name": "shares", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, { "stream": { "name": "total_follower_count", @@ -53,15 +35,6 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "ugc_posts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-linkedin-pages/main.py b/airbyte-integrations/connectors/source-linkedin-pages/main.py index d601480ca89df..57f710102bd1e 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/main.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/main.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # import sys @@ -9,4 +9,4 @@ if __name__ == "__main__": source = SourceLinkedinPages() - launch(source, sys.argv[1:]) \ No newline at end of file + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/setup.py b/airbyte-integrations/connectors/source-linkedin-pages/setup.py index bae2e6204f394..06721a7d3f41d 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/setup.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/setup.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/analytics.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/analytics.py index 9cbdc2d307919..363d6db1d1c1c 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/analytics.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/analytics.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # @@ -182,4 +182,4 @@ def merge_chunks(chunked_result: Iterable[Mapping[str, Any]], merge_by_key: str) result = [] for item in merged: result.append(merged.get(item)) - yield from result \ No newline at end of file + yield from result diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json index 8d06eb23cb2ce..ae8fc08757ca1 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -1,76 +1,30 @@ { - "type": [ - "null", - "object" - ], + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { - "elements": { - "type": [ - "null", - "array" - ], + "followerCountsByStaffCountRange": { + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], - "properties": { - "followerCountsByAssociationType": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByRegion": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsBySeniority": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByIndustry": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByFunction": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByStaffCountRange": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "followerCountsByCountry": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "organizationalEntity": { - "type": [ - "null", - "string" - ] - } - } + "type": ["null", "object"] + } + }, + "followerCountsByFunction": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"] + } + }, + "followerCountsByAssociationType": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"] + } + }, + "followerCountsBySeniority": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"] } } } diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json index 82ce72ca251f4..32a30f0625d92 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -1,8 +1,7 @@ { - "type": [ - "null", - "object" - ], + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { "vanityName": { "type": [ diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json deleted file mode 100644 index 248f4dcca47f5..0000000000000 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/page_statistics.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "paging": { - "type": ["null", "object"], - "properties": { - "start": { - "type": ["null", "integer"] - }, - "count": { - "type": ["null", "integer"] - }, - "links": { - "type": ["null", "array"], - "items": {} - } - } - }, - "elements": { - "type": ["null", "array"], - "items": {} - } - } -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json index 621997bd13b52..6f8b7077e116b 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json @@ -1,27 +1,40 @@ { - "type": ["null", "object"], + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { - "paging": { + "totalShareStatistics": { "type": ["null", "object"], "properties": { - "start": { + "uniqueImpressionsCount": { "type": ["null", "integer"] }, - "count": { + "clickCount": { "type": ["null", "integer"] }, - "links": { - "type": ["null", "array"], - "items": {} + "engagement": { + "type": ["null", "number"] }, - "total": { + "likeCount": { + "type": ["null", "integer"] + }, + "commentCount": { + "type": ["null", "integer"] + }, + "shareCount": { + "type": ["null", "integer"] + }, + "commentMentionsCount": { + "type": ["null", "integer"] + }, + "impressionCount": { + "type": ["null", "integer"] + }, + "shareMentionsCount": { "type": ["null", "integer"] } } }, - "elements": { - "type": ["null", "array"], - "items": {} - } + "organizationalEntity": {"type": ["null", "string"]} } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json deleted file mode 100644 index 621997bd13b52..0000000000000 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/shares.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "paging": { - "type": ["null", "object"], - "properties": { - "start": { - "type": ["null", "integer"] - }, - "count": { - "type": ["null", "integer"] - }, - "links": { - "type": ["null", "array"], - "items": {} - }, - "total": { - "type": ["null", "integer"] - } - } - }, - "elements": { - "type": ["null", "array"], - "items": {} - } - } -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json index d48ce8aa10a17..0c4a0c7d8c767 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json @@ -1,5 +1,7 @@ { - "type": ["null", "object"], + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { "firstDegreeSize": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json deleted file mode 100644 index 621997bd13b52..0000000000000 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/ugc_posts.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "paging": { - "type": ["null", "object"], - "properties": { - "start": { - "type": ["null", "integer"] - }, - "count": { - "type": ["null", "integer"] - }, - "links": { - "type": ["null", "array"], - "items": {} - }, - "total": { - "type": ["null", "integer"] - } - } - }, - "elements": { - "type": ["null", "array"], - "items": {} - } - } -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py index a8c80c5efc82c..c612dce735fa4 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py @@ -1,10 +1,10 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # from abc import ABC -from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests from airbyte_cdk import AirbyteLogger @@ -24,7 +24,6 @@ def __init__(self, config): super().__init__(authenticator=config.get("authenticator")) self.config = config - @property def org(self): """Property to return the user Organization Id from input""" @@ -38,10 +37,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, return None def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None ) -> Iterable[Mapping]: return [response.json()] @@ -57,62 +53,44 @@ def should_retry(self, response: requests.Response) -> bool: self.logger.error(error_message) return super().should_retry(response) -class OrganizationLookup(LinkedinPagesStream): +class OrganizationLookup(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: path = f"organizations/{self.org}" return path -class FollowerStatistics(LinkedinPagesStream): +class FollowerStatistics(LinkedinPagesStream): def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: path = f"organizationalEntityFollowerStatistics?q=organizationalEntity&organizationalEntity=urn:li:organization:{self.org}" return path -class PageStatistics(LinkedinPagesStream): - - def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None + ) -> Iterable[Mapping]: + yield from response.json().get("elements") - path = f"organizationPageStatistics?q=organization&organization=urn%3Ali%3Aorganization%3A{self.org}" - return path class ShareStatistics(LinkedinPagesStream): - def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: path = f"organizationalEntityShareStatistics?q=organizationalEntity&organizationalEntity=urn%3Ali%3Aorganization%3A{self.org}" return path -class Shares(LinkedinPagesStream): - - def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None + ) -> Iterable[Mapping]: + yield from response.json().get("elements") - path = f"shares?q=owners&owners=urn%3Ali%3Aorganization%3A{self.org}&sortBy=LAST_MODIFIED&sharesPerOwner=50" - return path class TotalFollowerCount(LinkedinPagesStream): - def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: path = f"networkSizes/urn:li:organization:{self.org}?edgeType=CompanyFollowedByMember" return path -class UgcPosts(LinkedinPagesStream): - - def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - - path = f"ugcPosts?q=authors&authors=List(urn%3Ali%3Aorganization%3A{self.org})&sortBy=LAST_MODIFIED&count=50" - return path - - def request_headers(self, stream_state: Mapping[str, Any], **kwargs) -> Mapping[str, Any]: - """ - If org_ids are specified as user's input from configuration, - we must use MODIFIED header: {'X-RestLi-Protocol-Version': '2.0.0'} - """ - return {"X-RestLi-Protocol-Version": "2.0.0"} if self.org else {} - class SourceLinkedinPages(AbstractSource): """ @@ -169,9 +147,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: return [ OrganizationLookup(config), FollowerStatistics(config), - PageStatistics(config), ShareStatistics(config), - Shares(config), TotalFollowerCount(config), - UgcPosts(config) ] diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py index 745ed95943ea2..4bb4685fd7a0a 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # import json @@ -321,4 +321,4 @@ def transform_data(records: List) -> Iterable[Mapping]: if "variables" in record: record = transform_variables(record) - yield record \ No newline at end of file + yield record diff --git a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_incremental_streams.py deleted file mode 100644 index 4b7913646e2ac..0000000000000 --- a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,59 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.models import SyncMode -from pytest import fixture -from source_linkedin_pages.source import IncrementalLinkedinPagesStream - - -@fixture -def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(IncrementalLinkedinPagesStream, "path", "v0/example_endpoint") - mocker.patch.object(IncrementalLinkedinPagesStream, "primary_key", "test_primary_key") - mocker.patch.object(IncrementalLinkedinPagesStream, "__abstractmethods__", set()) - - -def test_cursor_field(patch_incremental_base_class): - stream = IncrementalLinkedinPagesStream() - # TODO: replace this with your expected cursor field - expected_cursor_field = [] - assert stream.cursor_field == expected_cursor_field - - -def test_get_updated_state(patch_incremental_base_class): - stream = IncrementalLinkedinPagesStream() - # TODO: replace this with your input parameters - inputs = {"current_stream_state": None, "latest_record": None} - # TODO: replace this with your expected updated stream state - expected_state = {} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_stream_slices(patch_incremental_base_class): - stream = IncrementalLinkedinPagesStream() - # TODO: replace this with your input parameters - inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} - # TODO: replace this with your expected stream slices list - expected_stream_slice = [None] - assert stream.stream_slices(**inputs) == expected_stream_slice - - -def test_supports_incremental(patch_incremental_base_class, mocker): - mocker.patch.object(IncrementalLinkedinPagesStream, "cursor_field", "dummy_field") - stream = IncrementalLinkedinPagesStream() - assert stream.supports_incremental - - -def test_source_defined_cursor(patch_incremental_base_class): - stream = IncrementalLinkedinPagesStream() - assert stream.source_defined_cursor - - -def test_stream_checkpoint_interval(patch_incremental_base_class): - stream = IncrementalLinkedinPagesStream() - # TODO: replace this with your expected checkpoint interval - expected_checkpoint_interval = None - assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py deleted file mode 100644 index 5609fea61d95a..0000000000000 --- a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_source.py +++ /dev/null @@ -1,21 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from source_linkedin_pages.source import SourceLinkedinPages - - -def test_check_connection(mocker): - source = SourceLinkedinPages() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceLinkedinPages() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 7 - assert len(streams) == expected_streams_number \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_streams.py deleted file mode 100644 index 203fc70bb3dfa..0000000000000 --- a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/test_streams.py +++ /dev/null @@ -1,83 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_linkedin_pages.source import LinkedinPagesStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(LinkedinPagesStream, "path", "v0/example_endpoint") - mocker.patch.object(LinkedinPagesStream, "primary_key", "test_primary_key") - mocker.patch.object(LinkedinPagesStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = LinkedinPagesStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request parameters - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = LinkedinPagesStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected next page token - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class): - stream = LinkedinPagesStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected parced object - expected_parsed_object = {} - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = LinkedinPagesStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = LinkedinPagesStream() - # TODO: replace this with your expected http request method - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = LinkedinPagesStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = LinkedinPagesStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte_github_hg_social_repo_url b/airbyte_github_hg_social_repo_url deleted file mode 100644 index 6fce7b201a911..0000000000000 --- a/airbyte_github_hg_social_repo_url +++ /dev/null @@ -1,5 +0,0 @@ -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 - -https://airbyteuser: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@github.com/Hunt-Gather-Create/hg_social_media_analytics.git - -https://github.com/Hunt-Gather-Create/hg_social_media_analytics.git \ No newline at end of file From e953f235c9a0d30c68294fbf93ebb9da7e17b7fb Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 11 Aug 2022 11:22:34 -0300 Subject: [PATCH 14/21] remove gh file --- airbyte_github_hg_social_repo_url | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 airbyte_github_hg_social_repo_url diff --git a/airbyte_github_hg_social_repo_url b/airbyte_github_hg_social_repo_url deleted file mode 100644 index 6fce7b201a911..0000000000000 --- a/airbyte_github_hg_social_repo_url +++ /dev/null @@ -1,5 +0,0 @@ -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 - -https://airbyteuser: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@github.com/Hunt-Gather-Create/hg_social_media_analytics.git - -https://github.com/Hunt-Gather-Create/hg_social_media_analytics.git \ No newline at end of file From 8043f57be09adad9ccd3c4d29ae058586ab52828 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 11 Aug 2022 11:28:45 -0300 Subject: [PATCH 15/21] make org_id secret --- .../source-linkedin-pages/source_linkedin_pages/spec.json | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json index be64213a3de8f..4773b521f3f0f 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json @@ -10,6 +10,7 @@ "org_id": { "title": "Organization ID", "type": "integer", + "airbyte_secret": true, "description": "Specify the Organization ID", "examples": [123456789] }, From 3e0ea7a32447a96772a291cb94a9684f388b2fbe Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 11 Aug 2022 11:50:46 -0300 Subject: [PATCH 16/21] change org_id to string --- .../source-linkedin-pages/source_linkedin_pages/spec.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json index 4773b521f3f0f..ecc2acc04b10e 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json @@ -9,10 +9,10 @@ "properties": { "org_id": { "title": "Organization ID", - "type": "integer", + "type": "string", "airbyte_secret": true, "description": "Specify the Organization ID", - "examples": [123456789] + "examples": ["123456789"] }, "credentials": { "title": "Authentication *", From 6776baed39dc620b9c1a62a4ef48e07bff436a55 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 11 Aug 2022 13:13:43 -0300 Subject: [PATCH 17/21] rollback org_id --- .../source-linkedin-pages/source_linkedin_pages/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json index ecc2acc04b10e..085a281bdad3b 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json @@ -9,7 +9,7 @@ "properties": { "org_id": { "title": "Organization ID", - "type": "string", + "type": "integer", "airbyte_secret": true, "description": "Specify the Organization ID", "examples": ["123456789"] From 9cce62688352f43ef9e90662433c6d6c31ca9eec Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 11 Aug 2022 13:44:54 -0300 Subject: [PATCH 18/21] return org_id to integer --- .../source-linkedin-pages/source_linkedin_pages/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json index 085a281bdad3b..ecc2acc04b10e 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json @@ -9,7 +9,7 @@ "properties": { "org_id": { "title": "Organization ID", - "type": "integer", + "type": "string", "airbyte_secret": true, "description": "Specify the Organization ID", "examples": ["123456789"] From 57075609ddc631289be4c305f022effc7c09c78d Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 11 Aug 2022 14:44:19 -0300 Subject: [PATCH 19/21] rollback to integrr after correct ci --- .../source-linkedin-pages/source_linkedin_pages/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json index ecc2acc04b10e..085a281bdad3b 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json @@ -9,7 +9,7 @@ "properties": { "org_id": { "title": "Organization ID", - "type": "string", + "type": "integer", "airbyte_secret": true, "description": "Specify the Organization ID", "examples": ["123456789"] From 0f94d7cd61ff57878adaca73b2d3b0b3d48c9db4 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 11 Aug 2022 16:16:24 -0300 Subject: [PATCH 20/21] correct docs --- .../init/src/main/resources/seed/source_definitions.yaml | 8 ++++++++ airbyte-integrations/builds.md | 2 +- docs/integrations/sources/linkedin-pages.md | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index b44c0d4442493..107f98e1ffd7d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -520,6 +520,14 @@ icon: linkedin.svg sourceType: api releaseStage: generally_available +- name: LinkedIn Pages + sourceDefinitionId: af54297c-e8f8-4d63-a00d-a94695acc9d3 + dockerRepository: airbyte/source-linkedin-pages + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/linkedin-pages + icon: linkedin.svg + sourceType: api + releaseStage: alpha - name: Linnworks sourceDefinitionId: 7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e dockerRepository: airbyte/source-linnworks diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 6d57ebd743155..85ff8df3058c0 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -52,7 +52,7 @@ | Iterable | [![source-iterable](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-iterable%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-iterable) | | Jira | [![source-jira](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-jira%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-jira) | | LinkedIn Ads | [![source-linkedin-ads](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-linkedin-ads%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-linkedin-ads) | -| LinkedIn Pages | | +| LinkedIn Pages | [![source-linkedin-ads](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-linkedin-ads%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-linkedin-pages) | | Linnworks | [![source-linnworks](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-linnworks%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-linnworks) | | Lever Hiring | [![source-lever-hiring](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-lever-hiring%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-lever-hiring) | | Looker | [![source-looker](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-looker%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-looker) | diff --git a/docs/integrations/sources/linkedin-pages.md b/docs/integrations/sources/linkedin-pages.md index 743604a7fa14b..5801bcfff1f33 100644 --- a/docs/integrations/sources/linkedin-pages.md +++ b/docs/integrations/sources/linkedin-pages.md @@ -110,3 +110,4 @@ The source LinkedIn Pages can use either the `client_id`, `client_secret` and `r | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------- | +| 0.1.0 | 2022-08-11 | [13098](https://github.com/airbytehq/airbyte/pull/13098) | Initial Release | \ No newline at end of file From f0aa68b2fc5401ed08042a1364bfbc3ddb46b207 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Thu, 11 Aug 2022 19:37:32 +0000 Subject: [PATCH 21/21] auto-bump connector version [ci skip] --- .../src/main/resources/seed/source_specs.yaml | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 47ca656e05146..2eb3d36376ad9 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4718,6 +4718,84 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" +- dockerImage: "airbyte/source-linkedin-pages:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/linkedin-pages/" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Linkedin Pages Spec" + type: "object" + required: + - "org_id" + additionalProperties: true + properties: + org_id: + title: "Organization ID" + type: "integer" + airbyte_secret: true + description: "Specify the Organization ID" + examples: + - "123456789" + credentials: + title: "Authentication *" + type: "object" + oneOf: + - type: "object" + title: "OAuth2.0" + required: + - "client_id" + - "client_secret" + - "refresh_token" + properties: + auth_method: + type: "string" + const: "oAuth2.0" + client_id: + type: "string" + title: "Client ID" + description: "The client ID of the LinkedIn developer application." + airbyte_secret: true + client_secret: + type: "string" + title: "Client secret" + description: "The client secret of the LinkedIn developer application." + airbyte_secret: true + refresh_token: + type: "string" + title: "Refresh token" + description: "The token value generated using the LinkedIn Developers\ + \ OAuth Token Tools. See the docs to obtain yours." + airbyte_secret: true + - title: "Access token" + type: "object" + required: + - "access_token" + properties: + auth_method: + type: "string" + const: "access_token" + access_token: + type: "string" + title: "Access token" + description: "The token value generated using the LinkedIn Developers\ + \ OAuth Token Tools. See the docs to obtain yours." + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] + authSpecification: + auth_type: "oauth2.0" + oauth2Specification: + rootObject: + - "credentials" + - "0" + oauthFlowInitParameters: + - - "client_id" + - - "client_secret" + oauthFlowOutputParameters: + - - "refresh_token" - dockerImage: "airbyte/source-linnworks:0.1.5" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/linnworks"