Merge private and public id sets (demisto#10189)

* download private_id_set script, add to the build

* fix file name

* fix file name

* fix file name

* fix file name

* fix file name

* some tests

* some tests

* add service account

* add service account

* add GOOGLE_APPLICATION_CREDENTIALS

* add artifacts

* add artifacts

* xsoar-ci-artifacts

* merge id sets

* merge id sets

* change sdk to use merge id sets command

* change sdk to use merge id sets command

* change sdk to use merge id sets command

* add prepare_private_id_set_for_merge and upload_private_id_set scripts

* add test files

* add test

* add test

* fix test

* fix test

* fix test

* remove test

* add test, (remove changes from build)

* fix test and validation

* fix path

* try another bucket

* try another file

* some test

* some test

* some test

* service_account

* public_and_private_id_sets file

* remove public_and_private_id_sets file

* remove cp to artifacts

* remove cp to artifacts

* $CIRCLE_ARTIFACTS

* add file

* merge to id_set file

* CR changes

* CR changes

* fix validation

* fix validation

* change private id set path in bucket

* fix test, try download with gsutil

* try download with gsutil

* try download with gsutil

* try download with gsutil

* download with script

* try to decode gcs creds

* try with gsutil

* try with gsutil

* add file unified_id_sets

* remove file

* change path of unified id sets

* replace sdk branch

* add id_set_private_test file to bucket

* remove scripts use gsutil instead

* use another branch

* remove script from this PR

* add script and test

* remove ref for demisto sdk

* some fixes

* some fixes

* fix artifacts

* unified id set file

* fixes

* fixes

* open new file

* fixes

* fixes

* fixes

* try to download with script

* some fixes

* some fixes

* ref sdk

* fixes

* comment

* remove ref

* fixes

* try with prints

* add ref

* comment

* comment

* comment

* comment

* remove ref

* comment

* ref

* cache

* comment

* try to remove duplicates

* fix duplicate id

* fix duplicate ID issue

* add check that id sets merge successfully

* add check that id sets merge successfully

* fix duplicate

* add check in sdk command

* comment

* add duplicates for test

* fix duplicates for test

* duplicates for test

* typo

* fix duplicates - ready

* fix ref

* unified ID set just in validate

* check with ref sdk

* add id set path to validate script

* comment

* comment

* ref sdk to master

* add artifacts to run validation step

* add merge to create step

* fixes

* fixes

* ref to sdk validate mappers

* comment

* add step for merge id sets

* remove changes

* remove changes

* CR changes

.circleci/config.yml

@@ -311,6 +311,23 @@ references:
         demisto-sdk create-id-set -o ./Tests/id_set.json
         cp ./Tests/id_set.json $CIRCLE_ARTIFACTS
 
+  merge_public_and_private_id_sets: &merge_public_and_private_id_sets
+    run:
+      name: Merge public and private ID sets
+      when: always
+      command: |
+        GCS_PATH=$(mktemp)
+        echo "$GCS_MARKET_KEY" > "$GCS_PATH"
+        gcloud auth activate-service-account --key-file="$GCS_PATH" > auth.out 2>&1
+
+        # Download private ID set
+        gsutil cp gs://marketplace-dist/content/private_id_set.json $CIRCLE_ARTIFACTS/unified_id_set.json
+        gcloud auth revoke --all
+        rm $GCS_PATH
+
+        # Merge public and private ID sets
+        demisto-sdk merge-id-sets -i1 ./Tests/id_set.json -i2 $CIRCLE_ARTIFACTS/unified_id_set.json -o $CIRCLE_ARTIFACTS/unified_id_set.json
+
   build_content_descriptor: &build_content_descriptor
     run:
       name: Build Content Descriptor
@@ -545,6 +562,7 @@ jobs:
       - *prepare_environment
       - *secrets
       - *create_id_set
+      - *merge_public_and_private_id_sets
       - *update_tests_step
       - *validate_files_and_yaml
       - run:

Tests/scripts/validate.sh

@@ -5,8 +5,7 @@ echo "CIRCLE_BRANCH: $CIRCLE_BRANCH CI: $CI DEMISTO_README_VALIDATION: $DEMISTO_
 
 if [[ $CIRCLE_BRANCH = master ]] || [[ -n "${NIGHTLY}" ]] || [[ -n "${BUCKET_UPLOAD}" ]];
   then
-    demisto-sdk validate -a
+    demisto-sdk validate -a --id-set-path $CIRCLE_ARTIFACTS/unified_id_set.json
 else
-  demisto-sdk validate -g --post-commit
+  demisto-sdk validate -g --post-commit --id-set-path $CIRCLE_ARTIFACTS/unified_id_set.json
 fi
-