TDL-26523: Upgrade salesforce API version to v61 (#184)

* TDL-26523: Upgrade API version to v61 * Update sync canary test * Skip EventLogFile from tests * Update changelog md * Define constant for api version and pylint fix * Update Bulk API * Consider some streams forcefully as FULL_TABLE * Skip black listed objects during sync mode * Update tap tester for newly added streams * Fix pylint issue * Update tap-tester * Fix replication-key issue * Fix forced-replication-methods things * Update release minor version * Fix sync canary tests * Fix sync canary tests * Fix sync canary test * Fix tap-tester
singer-io · Nov 6, 2024 · 48754f7 · 48754f7
1 parent ebb3e5e
commit 48754f7
Show file tree

Hide file tree

Showing 12 changed files with 710 additions and 71 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -62,29 +62,25 @@ jobs:
           path: test_output/report.xml
       - store_artifacts:
           path: htmlcov
-  run_integration_tests:
+  integration_test:
+    parameters:
+      test_command:
+        type: string
     executor: docker-executor
-    parallelism: 8
     steps:
       - checkout
       - attach_workspace:
           at: /usr/local/share/virtualenvs
       - run:
           name: 'Run Integration Tests'
-          no_output_timeout: 30m
+          no_output_timeout: 1h
           command: |
             aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh
             source dev_env.sh
             mkdir /tmp/${CIRCLE_PROJECT_REPONAME}
             export STITCH_CONFIG_DIR=/tmp/${CIRCLE_PROJECT_REPONAME}
             source /usr/local/share/virtualenvs/tap-tester/bin/activate
-            circleci tests glob "tests/*.py" | circleci tests split > ./tests-to-run
-            if [ -s ./tests-to-run ]; then
-              for test_file in $(cat ./tests-to-run)
-              do
-                run-test --tap=${CIRCLE_PROJECT_REPONAME} $test_file
-              done
-            fi
+            << parameters.test_command >>
       - slack/notify-on-failure:
           only_for_branches: master
       - store_artifacts:
@@ -110,19 +106,210 @@ workflows:
             - tier-1-tap-user
           requires:
             - ensure_env
-      - run_integration_tests:
+      - integration_test:
+          name: "Discovery Test Bulk API"
           context:
             - circleci-user
             - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_discovery_bulk.py
           requires:
             - ensure_env
+      - integration_test:
+          name: "Discovery Test REST API"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_discovery_rest.py
+          requires:
+            - ensure_env
+      - integration_test:
+          name: "Automatic Fields Test Bulk API"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_automatic_fields_bulk.py
+          requires:
+            - ensure_env
+      - integration_test:
+          name: "All Fields Non Custom Fields Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_all_fields_non_custom.py
+          requires:
+            - "Discovery Test Bulk API"
+      - integration_test:
+          name: "Automatic Fields Test REST API"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_automatic_fields_rest.py
+          requires:
+            - "Discovery Test REST API"
+      - integration_test:
+          name: "All Fields Custom Fields Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_all_fields_custom.py
+          requires:
+            - "Automatic Fields Test Bulk API"
+      - integration_test:
+          name: "Full table Replication"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_full_table_replication.py
+          requires:
+            - "All Fields Non Custom Fields Test"
+      - integration_test:
+          name: "Bookmark Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_bookmarks.py
+          requires:
+            - "Automatic Fields Test REST API"
+      - integration_test:
+          name: "Custom Object Rest API"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_custom_objects_rest.py
+          requires:
+            - "All Fields Custom Fields Test"
+      - integration_test:
+          name: "Lookback Window Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_lookback_window.py
+          requires:
+            - "Full table Replication"
+      - integration_test:
+          name: "Select By Default Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_select_by_default.py
+          requires:
+            - "Bookmark Test"
+      - integration_test:
+          name: "Switch Replication Method Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_switch_rep_method_ft_incrmntl.py
+          requires:
+            - "Custom Object Rest API"
+      - integration_test:
+          name: "Api Total Quota Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_api_total_quota.py
+          requires:
+            - "Lookback Window Test"
+      - integration_test:
+          name: "Custom Object Bulk API"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_custom_objects_bulk.py
+          requires:
+            - "Select By Default Test"
+      - integration_test:
+          name: "Incremental Table Reset Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_incremental_table_reset.py
+          requires:
+            - "Switch Replication Method Test"
+      - integration_test:
+          name: "Api Per Run Total Quota Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_api_per_run_quota.py
+          requires:
+            - "Api Total Quota Test"
+      - integration_test:
+          name: "All Fields Non Custom Fields Rest Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_all_fields_non_custom_rest.py
+          requires:
+            - "Custom Object Bulk API"
+      - integration_test:
+          name: "All Fields Custom Fields Rest Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_all_fields_custom_rest.py
+          requires:
+            - "Incremental Table Reset Test"
+      - integration_test:
+          name: "Activate Version Messages Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_activate_version_messages.py
+          requires:
+            - "Api Per Run Total Quota Test"
+      - integration_test:
+          name: "Start Date Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_start_date.py
+          requires:
+            - "Api Per Run Total Quota Test"
+      - integration_test:
+          name: "Switch Replication Method(Incremental) Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_switch_rep_method_incrmntl_ft.py
+          requires:
+            -  "All Fields Non Custom Fields Test"
+      - integration_test:
+          name: "Sync Canary Test"
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          test_command: |-
+            run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_salesforce_sync_canary.py
+          requires:
+            - "All Fields Custom Fields Test"
       - build:
           context:
             - circleci-user
             - tier-1-tap-user
           requires:
             - run_pylint
-            - run_integration_tests
   build_daily:
     <<: *commit_jobs
     triggers:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+## 2.0.2
+  - Upgrade salesforce API version `52.0` to `61.0`, details [#184](https://github.com/singer-io/tap-salesforce/pulls)
+
 ## 2.0.1
   * Fix `_can_pk_chunk_job` condition, details: [#176](https://github.com/singer-io/tap-salesforce/pull/176)
 

diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 from setuptools import setup
 
 setup(name='tap-salesforce',
-      version='2.0.1',
+      version='2.1.0',
       description='Singer.io tap for extracting data from the Salesforce API',
       author='Stitch',
       url='https://singer.io',

diff --git a/tap_salesforce/__init__.py b/tap_salesforce/__init__.py
@@ -35,6 +35,17 @@
     'UriEvent',
     'LogoutEvent',
     'ReportEvent',
+    'PermissionSetEventStore',
+    'ListViewEvent',
+    'IdentityProviderEventStore',
+    'ApiEvent',
+    'BulkApiResultEventStore',
+    'IdentityVerificationEvent',
+    'LoginAsEvent',
+    'FileEventStore',
+    'ExternalEncryptionRootKey',
+    'ActivityFieldHistory',
+    'PendingOrderSummary'
 }
 
 def get_replication_key(sobject_name, fields):
@@ -61,6 +72,17 @@ def build_state(raw_state, catalog):
 
     for catalog_entry in catalog['streams']:
         tap_stream_id = catalog_entry['tap_stream_id']
+
+        if tap_stream_id in FORCED_FULL_TABLE:
+            for metadata_entry in catalog_entry['metadata']:
+                if metadata_entry['breadcrumb'] == []:
+                    metadata_entry['metadata']['forced-replication-method'] = {
+                        'replication-method': 'FULL_TABLE',
+                        'reason': 'No valid replication keys found from the Salesforce API'
+                    }
+                    metadata_entry['metadata'].pop('replication-key', None)
+                    LOGGER.info("Forcing FULL_TABLE replication for %s", tap_stream_id)
+                    break
         catalog_metadata = metadata.to_map(catalog_entry['metadata'])
         replication_method = catalog_metadata.get((), {}).get('replication-method')
 
@@ -307,6 +329,10 @@ def do_sync(sf, catalog, state):
             LOGGER.info("%s: Skipping - not selected", stream_name)
             continue
 
+        if stream_name in sf.get_blacklisted_objects():
+            LOGGER.info("%s: Skipping - blacklisted", stream_name)
+            continue
+
         if starting_stream:
             if starting_stream == stream_name:
                 LOGGER.info("%s: Resuming", stream_name)

diff --git a/tap_salesforce/salesforce/__init__.py b/tap_salesforce/salesforce/__init__.py
@@ -10,7 +10,7 @@
 from singer import metadata, metrics
 
 from tap_salesforce.salesforce.bulk import Bulk
-from tap_salesforce.salesforce.rest import Rest
+from tap_salesforce.salesforce.rest import Rest, API_VERSION
 from tap_salesforce.salesforce.exceptions import (
     TapSalesforceException,
     TapSalesforceQuotaExceededException)
@@ -66,11 +66,12 @@
 
 
 # The following objects are not supported by the bulk API.
-UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS = set(['AssetTokenEvent',
-                                               'AttachedContentNote',
-                                               'EventWhoRelation',
-                                               'QuoteTemplateRichTextData',
-                                               'TaskWhoRelation',
+UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS = set(['FieldSecurityClassification',
+                                               'WorkStepStatus',
+                                               'ShiftStatus',
+                                               'WorkOrderStatus',
+                                               'WorkOrderLineItemStatus',
+                                               'ServiceAppointmentStatus',
                                                'SolutionStatus',
                                                'ContractStatus',
                                                'RecentlyViewed',
@@ -111,6 +112,12 @@
                                            'FlowVariableView',
                                            'AppTabMember',
                                            'ColorDefinition',
+                                           'DatacloudDandBCompany', # Not filterable without a criteria.
+                                           'DatacloudAddress', # Transient queries are not implemented
+                                           'FlowTestView', # A filter on a reified column is required [FlowDefinitionViewId,DurableId]
+                                           'RelatedListColumnDefinition', # A filter on a reified column is required [RelatedListDefinitionId,DurableId],
+                                           'RelatedListDefinition', # A filter on a reified column is required [ParentEntityDefinitionId,DurableId],
+                                           'ApexTypeImplementor', # A filter on a reified column is required [InterfaceName,DurableId]
                                            'IconDefinition',])
 
 # The following objects are not supported by the query method being used.
@@ -200,7 +207,7 @@ def field_to_property_schema(field, mdata): # pylint:disable=too-many-branches
     return property_schema, mdata
 
 class Salesforce():
-    # pylint: disable=too-many-instance-attributes,too-many-arguments
+    # pylint: disable=too-many-instance-attributes,too-many-arguments,too-many-positional-arguments
     def __init__(self,
                  refresh_token=None,
                  token=None,
@@ -235,7 +242,7 @@ def __init__(self,
         self.rest_requests_attempted = 0
         self.jobs_completed = 0
         self.login_timer = None
-        self.data_url = "{}/services/data/v52.0/{}"
+        self.data_url = "{}/services/data/v{}.0/{}"
         self.pk_chunking = False
         self.lookback_window = lookback_window
 
@@ -276,7 +283,7 @@ def check_rest_quota_usage(self, headers):
                                                                        self.quota_percent_per_run)
             raise TapSalesforceQuotaExceededException(partial_message)
 
-    # pylint: disable=too-many-arguments
+    # pylint: disable=too-many-arguments,too-many-positional-arguments
     @backoff.on_exception(backoff.expo,
                           (requests.exceptions.ConnectionError, requests.exceptions.Timeout),
                           max_tries=10,
@@ -361,11 +368,11 @@ def describe(self, sobject=None):
         if sobject is None:
             endpoint = "sobjects"
             endpoint_tag = "sobjects"
-            url = self.data_url.format(self.instance_url, endpoint)
+            url = self.data_url.format(self.instance_url, API_VERSION, endpoint)
         else:
             endpoint = "sobjects/{}/describe".format(sobject)
             endpoint_tag = sobject
-            url = self.data_url.format(self.instance_url, endpoint)
+            url = self.data_url.format(self.instance_url, API_VERSION, endpoint)
 
         with metrics.http_request_timer("describe") as timer:
             timer.tags['endpoint'] = endpoint_tag