Skip to content

Commit 2485899

Browse files
NicoDucouNicoDucou
authored
Merge pull request #5934 from AngelFQC/BT21930
Plugin: Azure: Add options to user delta queries when syncing
2 parents 10b6d61 + 5e10316 commit 2485899

File tree

10 files changed

+233
-20
lines changed

10 files changed

+233
-20
lines changed

‎plugin/azure_active_directory/CHANGELOG.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
11
# Azure Active Directory Changelog
22

3+
## 2.5 - 2024-11-18
4+
5+
* Added new options to get the user and groups with delta query (or change tracking) when syncing with scripts.
6+
this requires manually doing the following changes to your database if you are upgrading from v2.4
7+
```sql
8+
CREATE TABLE azure_ad_sync_state (id INT AUTO_INCREMENT NOT NULL, title VARCHAR(255) NOT NULL, value LONGTEXT NOT NULL, created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL, PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8 COLLATE `utf8_unicode_ci` ENGINE = InnoDB;
9+
```
10+
311
## 2.4 - 2024-08-28
412

513
* Added a new user extra field to save the unique Azure ID (internal UID).

‎plugin/azure_active_directory/lang/dutch.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,7 @@
4646
$strings['tenant_id_help'] = 'Required to run scripts.';
4747
$strings['deactivate_nonexisting_users'] = 'Deactivate non-existing users';
4848
$strings['deactivate_nonexisting_users_help'] = 'Compare registered users in Chamilo with those in Azure and deactivate accounts in Chamilo that do not exist in Azure.';
49+
$strings['script_users_delta'] = 'Delta query for users';
50+
$strings['script_users_delta_help'] = 'Get newly created, updated, or deleted users without having to perform a full read of the entire user collection. By default, is <code>No</code>.';
51+
$strings['script_usergroups_delta'] = 'Delta query for usergroups';
52+
$strings['script_usergroups_delta_help'] = 'Get newly created, updated, or deleted groups, including group membership changes, without having to perform a full read of the entire group collection. By default, is <code>No</code>.';

‎plugin/azure_active_directory/lang/english.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,7 @@
4646
$strings['tenant_id_help'] = 'Required to run scripts.';
4747
$strings['deactivate_nonexisting_users'] = 'Deactivate non-existing users';
4848
$strings['deactivate_nonexisting_users_help'] = 'Compare registered users in Chamilo with those in Azure and deactivate accounts in Chamilo that do not exist in Azure.';
49+
$strings['script_users_delta'] = 'Delta query for users';
50+
$strings['script_users_delta_help'] = 'Get newly created, updated, or deleted users without having to perform a full read of the entire user collection. By default, is <code>No</code>.';
51+
$strings['script_usergroups_delta'] = 'Delta query for usergroups';
52+
$strings['script_usergroups_delta_help'] = 'Get newly created, updated, or deleted groups, including group membership changes, without having to perform a full read of the entire group collection. By default, is <code>No</code>.';

‎plugin/azure_active_directory/lang/french.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,7 @@
4646
$strings['tenant_id_help'] = 'Nécessaire pour exécuter des scripts.';
4747
$strings['deactivate_nonexisting_users'] = 'Deactivate non-existing users';
4848
$strings['deactivate_nonexisting_users_help'] = 'Compare registered users in Chamilo with those in Azure and deactivate accounts in Chamilo that do not exist in Azure.';
49+
$strings['script_users_delta'] = 'Requête delta pour les utilisateurs';
50+
$strings['script_users_delta_help'] = 'Get newly created, updated, or deleted users without having to perform a full read of the entire user collection. By default, is <code>No</code>.';
51+
$strings['script_usergroups_delta'] = 'Requête delta pour les groupes d\'utilisateurs';
52+
$strings['script_usergroups_delta_help'] = 'Get newly created, updated, or deleted groups, including group membership changes, without having to perform a full read of the entire group collection. By default, is <code>No</code>.';

‎plugin/azure_active_directory/lang/spanish.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,7 @@
4646
$strings['tenant_id_help'] = 'Necesario para ejecutar scripts.';
4747
$strings['deactivate_nonexisting_users'] = 'Desactivar usuarios no existentes';
4848
$strings['deactivate_nonexisting_users_help'] = 'Compara los usuarios registrados en Chamilo con los de Azure y desactiva las cuentas en Chamilo que no existan en Azure.';
49+
$strings['script_users_delta'] = 'Consula delta para usuarios';
50+
$strings['script_users_delta_help'] = 'Obtiene usuarios recién creados, actualizados o eliminados sin tener que realizar una lectura completa de toda la colección de usuarios. De forma predeterminada, es <code>No</code>.';
51+
$strings['script_usergroups_delta'] = 'Consulta delta para grupos de usuarios';
52+
$strings['script_usergroups_delta_help'] = 'Obtiene grupos recién creados, actualizados o eliminados, incluidos los cambios de membresía del grupo, sin tener que realizar una lectura completa de toda la colección de grupos. De forma predeterminada, es <code>No</code>';

‎plugin/azure_active_directory/src/AzureActiveDirectory.php‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
<?php
22
/* For license terms, see /license.txt */
33

4+
use Chamilo\PluginBundle\Entity\AzureActiveDirectory\AzureSyncState;
45
use Chamilo\UserBundle\Entity\User;
6+
use Doctrine\ORM\Tools\SchemaTool;
7+
use Doctrine\ORM\Tools\ToolsException;
58
use TheNetworg\OAuth2\Client\Provider\Azure;
69

710
/**
@@ -28,6 +31,8 @@ class AzureActiveDirectory extends Plugin
2831
public const SETTING_EXISTING_USER_VERIFICATION_ORDER = 'existing_user_verification_order';
2932
public const SETTING_TENANT_ID = 'tenant_id';
3033
public const SETTING_DEACTIVATE_NONEXISTING_USERS = 'deactivate_nonexisting_users';
34+
public const SETTING_GET_USERS_DELTA = 'script_users_delta';
35+
public const SETTING_GET_USERGROUPS_DELTA = 'script_usergroups_delta';
3136

3237
public const URL_TYPE_AUTHORIZE = 'login';
3338
public const URL_TYPE_LOGOUT = 'logout';
@@ -59,9 +64,11 @@ protected function __construct()
5964
self::SETTING_EXISTING_USER_VERIFICATION_ORDER => 'text',
6065
self::SETTING_TENANT_ID => 'text',
6166
self::SETTING_DEACTIVATE_NONEXISTING_USERS => 'boolean',
67+
self::SETTING_GET_USERS_DELTA => 'boolean',
68+
self::SETTING_GET_USERGROUPS_DELTA => 'boolean',
6269
];
6370

64-
parent::__construct('2.4', 'Angel Fernando Quiroz Campos, Yannick Warnier', $settings);
71+
parent::__construct('2.5', 'Angel Fernando Quiroz Campos, Yannick Warnier', $settings);
6572
}
6673

6774
/**
@@ -131,6 +138,8 @@ public function getUrl($urlType)
131138

132139
/**
133140
* Create extra fields for user when installing.
141+
*
142+
* @throws ToolsException
134143
*/
135144
public function install()
136145
{
@@ -152,6 +161,35 @@ public function install()
152161
$this->get_lang('AzureUid'),
153162
''
154163
);
164+
165+
$em = Database::getManager();
166+
167+
if ($em->getConnection()->getSchemaManager()->tablesExist(['course_home_notify_notification'])) {
168+
return;
169+
}
170+
171+
$schemaTool = new SchemaTool($em);
172+
$schemaTool->createSchema(
173+
[
174+
$em->getClassMetadata(AzureSyncState::class),
175+
]
176+
);
177+
}
178+
179+
public function uninstall()
180+
{
181+
$em = Database::getManager();
182+
183+
if (!$em->getConnection()->getSchemaManager()->tablesExist(['course_home_notify_notification'])) {
184+
return;
185+
}
186+
187+
$schemaTool = new SchemaTool($em);
188+
$schemaTool->dropSchema(
189+
[
190+
$em->getClassMetadata(AzureSyncState::class),
191+
]
192+
);
155193
}
156194

157195
public function getExistingUserVerificationOrder(): array
@@ -385,4 +423,27 @@ private function formatUserData(
385423
$extra,
386424
];
387425
}
426+
427+
public function getSyncState(string $title): ?AzureSyncState
428+
{
429+
$stateRepo = Database::getManager()->getRepository(AzureSyncState::class);
430+
431+
return $stateRepo->findOneBy(['title' => $title]);
432+
}
433+
434+
public function saveSyncState(string $title, $value)
435+
{
436+
$state = $this->getSyncState($title);
437+
438+
if (!$state) {
439+
$state = new AzureSyncState();
440+
$state->setTitle($title);
441+
442+
Database::getManager()->persist($state);
443+
}
444+
445+
$state->setValue($value);
446+
447+
Database::getManager()->flush();
448+
}
388449
}

‎plugin/azure_active_directory/src/AzureCommand.php‎

Lines changed: 55 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
/* For license terms, see /license.txt */
44

5+
use Chamilo\PluginBundle\Entity\AzureActiveDirectory\AzureSyncState;
56
use League\OAuth2\Client\Provider\Exception\IdentityProviderException;
67
use League\OAuth2\Client\Token\AccessTokenInterface;
78
use TheNetworg\OAuth2\Client\Provider\Azure;
@@ -56,11 +57,21 @@ protected function getAzureUsers(): Generator
5657
'id',
5758
];
5859

59-
$query = sprintf(
60-
'$top=%d&$select=%s',
61-
AzureActiveDirectory::API_PAGE_SIZE,
62-
implode(',', $userFields)
63-
);
60+
$getUsersDelta = 'true' === $this->plugin->get(AzureActiveDirectory::SETTING_GET_USERS_DELTA);
61+
62+
if ($getUsersDelta) {
63+
$usersDeltaLink = $this->plugin->getSyncState(AzureSyncState::USERS_DATALINK);
64+
65+
$query = $usersDeltaLink
66+
? $usersDeltaLink->getValue()
67+
: sprintf('$select=%s', implode(',', $userFields));
68+
} else {
69+
$query = sprintf(
70+
'$top=%d&$select=%s',
71+
AzureActiveDirectory::API_PAGE_SIZE,
72+
implode(',', $userFields)
73+
);
74+
}
6475

6576
$token = null;
6677

@@ -70,7 +81,7 @@ protected function getAzureUsers(): Generator
7081
try {
7182
$azureUsersRequest = $this->provider->request(
7283
'get',
73-
"users?$query",
84+
$getUsersDelta ? "users/delta?$query" : "users?$query",
7485
$token
7586
);
7687
} catch (Exception $e) {
@@ -80,6 +91,10 @@ protected function getAzureUsers(): Generator
8091
$azureUsersInfo = $azureUsersRequest['value'] ?? [];
8192

8293
foreach ($azureUsersInfo as $azureUserInfo) {
94+
$azureUserInfo['mail'] = $azureUserInfo['mail'] ?? null;
95+
$azureUserInfo['surname'] = $azureUserInfo['surname'] ?? null;
96+
$azureUserInfo['givenName'] = $azureUserInfo['givenName'] ?? null;
97+
8398
yield $azureUserInfo;
8499
}
85100

@@ -89,6 +104,13 @@ protected function getAzureUsers(): Generator
89104
$hasNextLink = true;
90105
$query = parse_url($azureUsersRequest['@odata.nextLink'], PHP_URL_QUERY);
91106
}
107+
108+
if ($getUsersDelta && !empty($azureUsersRequest['@odata.deltaLink'])) {
109+
$this->plugin->saveSyncState(
110+
AzureSyncState::USERS_DATALINK,
111+
parse_url($azureUsersRequest['@odata.deltaLink'], PHP_URL_QUERY),
112+
);
113+
}
92114
} while ($hasNextLink);
93115
}
94116

@@ -105,19 +127,33 @@ protected function getAzureGroups(): Generator
105127
'description',
106128
];
107129

108-
$query = sprintf(
109-
'$top=%d&$select=%s',
110-
AzureActiveDirectory::API_PAGE_SIZE,
111-
implode(',', $groupFields)
112-
);
130+
$getUsergroupsDelta = 'true' === $this->plugin->get(AzureActiveDirectory::SETTING_GET_USERGROUPS_DELTA);
131+
132+
if ($getUsergroupsDelta) {
133+
$usergroupsDeltaLink = $this->plugin->getSyncState(AzureSyncState::USERGROUPS_DATALINK);
134+
135+
$query = $usergroupsDeltaLink
136+
? $usergroupsDeltaLink->getValue()
137+
: sprintf('$select=%s', implode(',', $groupFields));
138+
} else {
139+
$query = sprintf(
140+
'$top=%d&$select=%s',
141+
AzureActiveDirectory::API_PAGE_SIZE,
142+
implode(',', $groupFields)
143+
);
144+
}
113145

114146
$token = null;
115147

116148
do {
117149
$this->generateOrRefreshToken($token);
118150

119151
try {
120-
$azureGroupsRequest = $this->provider->request('get', "groups?$query", $token);
152+
$azureGroupsRequest = $this->provider->request(
153+
'get',
154+
$getUsergroupsDelta ? "groups/delta?$query" : "groups?$query",
155+
$token
156+
);
121157
} catch (Exception $e) {
122158
throw new Exception('Exception when requesting groups from Azure: '.$e->getMessage());
123159
}
@@ -134,6 +170,13 @@ protected function getAzureGroups(): Generator
134170
$hasNextLink = true;
135171
$query = parse_url($azureGroupsRequest['@odata.nextLink'], PHP_URL_QUERY);
136172
}
173+
174+
if ($getUsergroupsDelta && !empty($azureGroupsRequest['@odata.deltaLink'])) {
175+
$this->plugin->saveSyncState(
176+
AzureSyncState::USERGROUPS_DATALINK,
177+
parse_url($azureGroupsRequest['@odata.deltaLink'], PHP_URL_QUERY),
178+
);
179+
}
137180
} while ($hasNextLink);
138181
}
139182

‎plugin/azure_active_directory/src/AzureSyncUsersCommand.php‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ public function __invoke(): Generator
1515
{
1616
yield 'Synchronizing users from Azure.';
1717

18-
/** @var array<string, int> $existingUsers */
19-
$existingUsers = [];
18+
/** @var array<string, int> $azureCreatedUserIdList */
19+
$azureCreatedUserIdList = [];
2020

2121
foreach ($this->getAzureUsers() as $azureUserInfo) {
2222
try {
@@ -27,7 +27,7 @@ public function __invoke(): Generator
2727
continue;
2828
}
2929

30-
$existingUsers[$azureUserInfo['id']] = $userId;
30+
$azureCreatedUserIdList[$azureUserInfo['id']] = $userId;
3131

3232
yield sprintf('User (ID %d) with received info: %s ', $userId, serialize($azureUserInfo));
3333
}
@@ -53,7 +53,7 @@ public function __invoke(): Generator
5353
$azureGroupMembersUids = array_column($azureGroupMembersInfo, 'id');
5454

5555
foreach ($azureGroupMembersUids as $azureGroupMembersUid) {
56-
$userId = $existingUsers[$azureGroupMembersUid] ?? null;
56+
$userId = $azureCreatedUserIdList[$azureGroupMembersUid] ?? null;
5757

5858
if (!$userId) {
5959
continue;
@@ -72,20 +72,22 @@ public function __invoke(): Generator
7272
$em->flush();
7373
}
7474

75-
if ('true' === $this->plugin->get(AzureActiveDirectory::SETTING_DEACTIVATE_NONEXISTING_USERS)) {
75+
if ('true' === $this->plugin->get(AzureActiveDirectory::SETTING_DEACTIVATE_NONEXISTING_USERS)
76+
&& 'true' !== $this->plugin->get(AzureActiveDirectory::SETTING_GET_USERS_DELTA)
77+
) {
7678
yield '----------------';
7779

7880
yield 'Trying deactivate non-existing users in Azure';
7981

8082
$users = UserManager::getRepository()->findByAuthSource('azure');
81-
$userIdList = array_map(
83+
$chamiloUserIdList = array_map(
8284
function ($user) {
8385
return $user->getId();
8486
},
8587
$users
8688
);
8789

88-
$nonExistingUsers = array_diff($userIdList, $existingUsers);
90+
$nonExistingUsers = array_diff($chamiloUserIdList, $azureCreatedUserIdList);
8991

9092
UserManager::deactivate_users($nonExistingUsers);
9193

0 commit comments

Comments
 (0)