Merged PR 56874: Do not sync Email, EmailType or OptInType as data fi…

…elds

## What's being changed

This PR fixes handling of some data points previously treated as data fields in the context of CSV sync.

All contact exports now do not export Email or EmailType. Subscriber exports now set OptInType per subscriber depending on Customers > Newsletter > Subscription Options > Need to Confirm at _store level_ (this is a change - we used to check this at website level but the config is store-scoped).

## Why it's being changed

Email, EmailType and OptInType are not data fields - in V3, EmailType and OptInType are channel properties, and in fact even in V2, these data points are sent separately from data fields in API requests. It's only because of how we used to send these via CSV that they were handled the way they were.

## How to review / test this change

- Run customer sync, guest sync and subscriber sync
- All exported JSON should have `,"channelProperties":{"email":{"emailType":"html"}}` for each contact
- Subscriber exports should _not_ contain `optInType` yet
- Turn on Customers > Newsletter > Subscription Options > Need to Confirm in a store scope
- Re-sync subscribers
- Subscribers in the affected store scope should be synced with `"channelProperties":{"email":{"emailType":"html","optInType":"double"}}`
- Email, EmailType and OptInType should not be seen in data fields at all

Related work items: #264085, #265397, #265398

Model/Apiconnector/Client.php

@@ -274,6 +274,9 @@ public function postAddressBookContacts($addressBookId, $apiContact)
      *
      * @return object
      * @throws LocalizedException
+     *
+     * @deprecated Use API V3.
+     * @see \Dotdigital\V3\Resources\Contacts::patchByIdentifier
      */
     public function addContactToAddressBook(
         string $email,

Model/Connector/ContactData.php

@@ -194,10 +194,6 @@ public function setContactData()
                         : null;
                     break;
 
-                case 'email_type':
-                    $value = 'Html';
-                    break;
-
                 default:
                     $method = 'get' . str_replace(' ', '', ucwords(str_replace('_', ' ', $key)));
                     $value = method_exists($this, $method)

Model/Connector/ContactData/Customer.php

@@ -172,6 +172,9 @@ public function init(AbstractModel $model, array $columns)
      * @param string $email
      *
      * @return void
+     *
+     * @deprecated email is an identifier, not a data field.
+     * @see \Dotdigitalgroup\Email\Model\Sync\Export\SdkContactBuilder::createSdkContact
      */
     public function setEmail($email)
     {
@@ -184,6 +187,9 @@ public function setEmail($email)
      * @param string $emailType
      *
      * @return void
+     *
+     * @deprecated email_type is a channel property, not a data field.
+     * @see \Dotdigitalgroup\Email\Model\Sync\Export\SdkContactBuilder::createSdkContact
      */
     public function setEmailType($emailType)
     {

Model/Connector/ContactData/Subscriber.php

@@ -4,13 +4,20 @@
 
 use Dotdigitalgroup\Email\Model\Connector\ContactData;
 
+/**
+ * @deprecated This class will be removed.
+ * @see \Dotdigitalgroup\Email\Model\Connector\ContactData;
+ */
 class Subscriber extends ContactData
 {
     /**
      * Get opt in type.
      *
      * If we are requesting a value for this column, it means 'Need to confirm'
      * is switched on in Magento.
+     *
+     * @deprecated OptInType is not a data field.
+     * @see \Dotdigitalgroup\Email\Model\Newsletter\OptInTypeFinder
      */
     public function getOptInType()
     {

Model/Newsletter/OptInTypeFinder.php

@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Dotdigitalgroup\Email\Model\Newsletter;
+
+use Magento\Framework\App\Config\ScopeConfigInterface;
+use Magento\Newsletter\Model\Subscriber;
+use Magento\Store\Model\ScopeInterface;
+
+class OptInTypeFinder
+{
+    /**
+     * @var ScopeConfigInterface
+     */
+    private $scopeConfig;
+
+    /**
+     * @param ScopeConfigInterface $scopeConfig
+     */
+    public function __construct(
+        ScopeConfigInterface $scopeConfig
+    ) {
+        $this->scopeConfig = $scopeConfig;
+    }
+
+    /**
+     * Get opt in type by store.
+     *
+     * @param int $storeId
+     *
+     * @return string|null
+     */
+    public function getOptInType($storeId)
+    {
+        $needToConfirm = $this->scopeConfig->getValue(
+            Subscriber::XML_PATH_CONFIRMATION_FLAG,
+            ScopeInterface::SCOPE_STORE,
+            $storeId
+        );
+
+        return $needToConfirm ? 'double' : null;
+    }
+}

Model/Sync/Automation/DataField/DataFieldCollector.php

@@ -186,13 +186,7 @@ public function collectForSubscriber(Contact $contact, $websiteId, int $listId):
     public function mergeFields(array $originalDataFields, array $newDataFields): array
     {
         $combinedDataFields = $originalDataFields;
-        $originalKeys = array_merge(
-            array_column($originalDataFields, 'Key'),
-            [
-                'Email',
-                'EmailType'
-            ]
-        );
+        $originalKeys = array_column($originalDataFields, 'Key');
 
         foreach ($newDataFields as $newDataField) {
             if (in_array($newDataField->getKey(), $originalKeys)) {

Model/Sync/Customer/Exporter.php

@@ -228,8 +228,7 @@ public function setFieldMapping(WebsiteInterface $website): void
             array_column($customAttributes, 'datafield')
         );
 
-        $columns = AbstractExporter::EMAIL_FIELDS + $customerDataFields;
-        $this->fieldMap = $attributeColumns ? $columns + $attributeColumns : $columns;
+        $this->fieldMap = $attributeColumns ? $customerDataFields + $attributeColumns : $customerDataFields;
     }
 
     /**

Model/Sync/Export/SdkContactBuilder.php

@@ -5,6 +5,7 @@
 namespace Dotdigitalgroup\Email\Model\Sync\Export;
 
 use Dotdigital\V3\Models\Contact as SdkContact;
+use Dotdigital\V3\Models\Contact\ChannelProperties\EmailChannelProperty;
 use Dotdigitalgroup\Email\Model\Contact;
 use Dotdigitalgroup\Email\Model\Connector\ContactData;
 use Dotdigitalgroup\Email\Model\Sync\Subscriber;
@@ -31,14 +32,16 @@ public function __construct(
      * @param ContactData $connectorModel
      * @param array $columns
      * @param int $listId
+     * @param string|null $optInType
      *
      * @return SdkContact
      * @throws \Exception
      */
     public function createSdkContact(
         ContactData $connectorModel,
         array $columns,
-        int $listId
+        int $listId,
+        ?string $optInType = null
     ): SdkContact {
         $sdkContact = new SdkContact([
             'matchIdentifier' => 'email'
@@ -48,6 +51,16 @@ public function createSdkContact(
         ]);
         $sdkContact->setLists([$listId]);
 
+        $emailChannelProperty = new EmailChannelProperty([
+            'emailType' => 'html'
+        ]);
+        if ($optInType) {
+            $emailChannelProperty->setOptInType($optInType);
+        }
+        $sdkContact->setChannelProperties([
+            'email' => $emailChannelProperty
+        ]);
+
         $sdkContact->setDataFields(
             $this->dataFieldMapper->mapFields(
                 $connectorModel->getContactData(),

Model/Sync/Guest/GuestExporter.php

@@ -116,7 +116,7 @@ public function setFieldMapping(WebsiteInterface $website): void
             'website_name' => $website->getConfig(Config::XML_PATH_CONNECTOR_CUSTOMER_WEBSITE_NAME)
         ];
 
-        $this->fieldMap = AbstractExporter::EMAIL_FIELDS + array_filter($guestColumns);
+        $this->fieldMap = array_filter($guestColumns);
     }
 
     /**

Model/Sync/Subscriber/SingleSubscriberSyncer.php

@@ -6,6 +6,7 @@
 
 use Dotdigitalgroup\Email\Helper\Data;
 use Dotdigitalgroup\Email\Model\Contact;
+use Dotdigitalgroup\Email\Model\Newsletter\OptInTypeFinder;
 use Dotdigitalgroup\Email\Model\Sync\Automation\DataField\DataFieldCollector;
 use Magento\Framework\Exception\LocalizedException;
 
@@ -21,16 +22,24 @@ class SingleSubscriberSyncer
      */
     private $dataFieldCollector;
 
+    /**
+     * @var OptInTypeFinder
+     */
+    private $optInTypeFinder;
+
     /**
      * @param Data $helper
      * @param DataFieldCollector $dataFieldCollector
+     * @param OptInTypeFinder $optInTypeFinder
      */
     public function __construct(
         Data $helper,
-        DataFieldCollector $dataFieldCollector
+        DataFieldCollector $dataFieldCollector,
+        OptInTypeFinder $optInTypeFinder
     ) {
         $this->helper = $helper;
         $this->dataFieldCollector = $dataFieldCollector;
+        $this->optInTypeFinder = $optInTypeFinder;
     }
 
     /**
@@ -60,11 +69,10 @@ public function pushContactToSubscriberAddressBook(Contact $contact)
             )
         );
 
-        // optInType will be set in $subscriberDataFields if it is 'Double'
         return $client->addContactToAddressBook(
             $contact->getEmail(),
             $subscriberAddressBookId,
-            null,
+            $this->optInTypeFinder->getOptInType($contact->getStoreId()),
             $subscriberDataFields
         );
     }

Model/Sync/Subscriber/SubscriberExporter.php

@@ -8,6 +8,7 @@
 use Dotdigitalgroup\Email\Helper\Config;
 use Dotdigitalgroup\Email\Logger\Logger;
 use Dotdigitalgroup\Email\Model\Connector\ContactData\SubscriberFactory as ConnectorSubscriberFactory;
+use Dotdigitalgroup\Email\Model\Newsletter\OptInTypeFinder;
 use Dotdigitalgroup\Email\Model\ResourceModel\Contact\CollectionFactory as ContactCollectionFactory;
 use Dotdigitalgroup\Email\Model\Sync\AbstractExporter;
 use Dotdigitalgroup\Email\Model\Sync\Export\CsvHandler;
@@ -27,6 +28,11 @@ class SubscriberExporter extends AbstractExporter implements ExporterInterface
      */
     private $connectorSubscriberFactory;
 
+    /**
+     * @var OptInTypeFinder
+     */
+    private $optInTypeFinder;
+
     /**
      * @var ContactCollectionFactory
      */
@@ -45,19 +51,22 @@ class SubscriberExporter extends AbstractExporter implements ExporterInterface
     /**
      * @param Logger $logger
      * @param ConnectorSubscriberFactory $connectorSubscriberFactory
+     * @param OptInTypeFinder $optInTypeFinder
      * @param ContactCollectionFactory $contactCollectionFactory
      * @param CsvHandler $csvHandler
      * @param SdkContactBuilder $sdkContactBuilder
      */
     public function __construct(
         Logger $logger,
         ConnectorSubscriberFactory $connectorSubscriberFactory,
+        OptInTypeFinder $optInTypeFinder,
         ContactCollectionFactory $contactCollectionFactory,
         CsvHandler $csvHandler,
         SdkContactBuilder $sdkContactBuilder
     ) {
         $this->logger = $logger;
         $this->connectorSubscriberFactory = $connectorSubscriberFactory;
+        $this->optInTypeFinder = $optInTypeFinder;
         $this->contactCollectionFactory = $contactCollectionFactory;
         $this->sdkContactBuilder = $sdkContactBuilder;
         parent::__construct($csvHandler);
@@ -93,7 +102,8 @@ public function export(array $subscribers, WebsiteInterface $website, int $listI
                 $exportedData[$subscriber->getId()] = $this->sdkContactBuilder->createSdkContact(
                     $connectorSubscriber,
                     $this->fieldMap,
-                    $listId
+                    $listId,
+                    $this->optInTypeFinder->getOptInType($subscriber->getStoreId())
                 );
 
                 $subscriber->clearInstance();
@@ -163,11 +173,7 @@ public function setFieldMapping(WebsiteInterface $website)
             'subscriber_status' => $website->getConfig(Config::XML_PATH_CONNECTOR_CUSTOMER_SUBSCRIBER_STATUS)
         ];
 
-        $this->fieldMap = AbstractExporter::EMAIL_FIELDS + array_filter($subscriberDataFields);
-
-        if ($this->isOptInTypeDouble($website)) {
-            $this->fieldMap += ['opt_in_type' => 'OptInType'];
-        }
+        $this->fieldMap = array_filter($subscriberDataFields);
     }
 
     /**
@@ -186,6 +192,9 @@ public function getFieldMapping(): array
      * @param WebsiteInterface $website
      *
      * @return boolean
+     *
+     * @deprecated OptInType is not a data field.
+     * @see \Dotdigitalgroup\Email\Model\Sync\Subscriber::loopByWebsite()
      */
     private function isOptInTypeDouble($website)
     {

Model/Sync/Subscriber/SubscriberWithSalesExporter.php

@@ -8,6 +8,7 @@
 use Dotdigitalgroup\Email\Logger\Logger;
 use Dotdigitalgroup\Email\Model\Connector\ContactData\SubscriberFactory as ConnectorSubscriberFactory;
 use Dotdigitalgroup\Email\Model\Connector\Datafield;
+use Dotdigitalgroup\Email\Model\Newsletter\OptInTypeFinder;
 use Dotdigitalgroup\Email\Model\ResourceModel\Contact\CollectionFactory as ContactCollectionFactory;
 use Dotdigitalgroup\Email\Model\Sync\AbstractExporter;
 use Dotdigitalgroup\Email\Model\Sync\Export\CsvHandler;
@@ -16,6 +17,7 @@
 use Dotdigitalgroup\Email\Model\Sync\Export\SdkContactBuilder;
 use Magento\Framework\App\Config\ScopeConfigInterface;
 use Magento\Framework\Exception\LocalizedException;
+use Magento\Newsletter\Model\Subscriber;
 use Magento\Store\Api\Data\WebsiteInterface;
 use Magento\Store\Model\ScopeInterface;
 
@@ -36,6 +38,11 @@ class SubscriberWithSalesExporter extends AbstractExporter implements ExporterIn
      */
     private $connectorSubscriberFactory;
 
+    /**
+     * @var OptInTypeFinder
+     */
+    private $optInTypeFinder;
+
     /**
      * @var ContactCollectionFactory
      */
@@ -70,6 +77,7 @@ class SubscriberWithSalesExporter extends AbstractExporter implements ExporterIn
      * @param Logger $logger
      * @param Datafield $datafield
      * @param ConnectorSubscriberFactory $connectorSubscriberFactory
+     * @param OptInTypeFinder $optInTypeFinder
      * @param ContactCollectionFactory $contactCollectionFactory
      * @param SalesDataManager $salesDataManager
      * @param SdkContactBuilder $sdkContactBuilder
@@ -81,6 +89,7 @@ public function __construct(
         Logger $logger,
         Datafield $datafield,
         ConnectorSubscriberFactory $connectorSubscriberFactory,
+        OptInTypeFinder $optInTypeFinder,
         ContactCollectionFactory $contactCollectionFactory,
         SalesDataManager $salesDataManager,
         SdkContactBuilder $sdkContactBuilder,
@@ -91,6 +100,7 @@ public function __construct(
         $this->logger = $logger;
         $this->datafield = $datafield;
         $this->connectorSubscriberFactory = $connectorSubscriberFactory;
+        $this->optInTypeFinder = $optInTypeFinder;
         $this->contactCollectionFactory = $contactCollectionFactory;
         $this->salesDataManager = $salesDataManager;
         $this->sdkContactBuilder = $sdkContactBuilder;
@@ -139,7 +149,8 @@ public function export(array $subscribers, WebsiteInterface $website, int $listI
                 $exportedData[$subscriber->getId()] = $this->sdkContactBuilder->createSdkContact(
                     $connectorSubscriber,
                     $this->fieldMap,
-                    $listId
+                    $listId,
+                    $this->optInTypeFinder->getOptInType($subscriber->getStoreId())
                 );
 
                 $subscriber->clearInstance();

Test/Unit/Model/Sync/Automation/DataField/DataFieldCollectorTest.php

@@ -304,8 +304,6 @@ private function getDefaultDataFields()
     private function getExportedCustomerDataFields()
     {
         return [
-            new DataField('Email', 'chaz@emailsim.io'),
-            new DataField('EmailType', 'Html'),
             new DataField('FIRST_NAME', 'Chaz'),
             new DataField('LAST_NAME', 'Kangaroo'),
             new DataField('SUBSCRIBER_STATUS', 'Subscribed'),