diff --git a/app/actions/remote/channel.test.ts b/app/actions/remote/channel.test.ts index 4a86cb6b8b0..cf78388e46c 100644 --- a/app/actions/remote/channel.test.ts +++ b/app/actions/remote/channel.test.ts @@ -105,13 +105,13 @@ const mockClient = { getUser: jest.fn((userId: string) => ({...user, id: userId})), getMyChannels: jest.fn((teamId: string) => ([{id: channelId, name: 'channel1', creatorId: user.id, team_id: teamId}])), getMyChannelMembers: jest.fn(() => ([{id: user.id + '-' + channelId, user_id: user.id, channel_id: channelId, roles: ''}])), - getCategories: jest.fn((userId: string, teamId: string) => ({categories: [{id: 'categoryid', channel_id: [channelId], team_id: teamId}], order: ['categoryid']})), + getCategories: jest.fn((userId: string, teamId: string) => ({categories: [{id: 'categoryid', channel_ids: [channelId], team_id: teamId}], order: ['categoryid']})), viewMyChannel: jest.fn(), getTeamByName: jest.fn((teamName: string) => ({id: teamId, name: teamName})), getTeam: jest.fn((id: string) => ({id, name: 'teamname'})), addToTeam: jest.fn((teamId: string, userId: string) => ({id: userId + '-' + teamId, user_id: userId, team_id: teamId, roles: ''})), getUserByUsername: jest.fn((username: string) => ({...user, id: 'userid2', username})), - createDirectChannel: jest.fn((userId1: string, userId2: string) => ({id: userId1 + '__' + userId2, team_id: '', type: 'D', display_name: 'displayname'})), + createDirectChannel: jest.fn((userIds: string[]) => ({id: userIds[0] + '__' + userIds[1], team_id: '', type: 'D', display_name: 'displayname'})), getChannels: jest.fn((teamId: string) => ([{id: channelId, name: 'channel1', creatorId: user.id, team_id: teamId}])), getArchivedChannels: jest.fn((teamId: string) => ([{id: channelId + 'old', name: 'channel1old', creatorId: user.id, team_id: teamId, delete_at: 1}])), createGroupChannel: jest.fn(() => ({id: 'groupid', team_id: '', type: 'G', display_name: 'displayname'})), @@ -129,6 +129,11 @@ const mockClient = { convertChannelToPrivate: jest.fn(), getGroupMessageMembersCommonTeams: jest.fn(() => ({id: teamId, name: 'teamname'})), convertGroupMessageToPrivateChannel: jest.fn((channelId: string) => ({id: channelId, name: 'channel1', creatorId: user.id, type: 'P'})), + getPosts: jest.fn(() => ({ + order: [], + posts: [], + previousPostId: '', + })), }; const teamId = 'teamid1'; diff --git a/app/actions/remote/channel.ts b/app/actions/remote/channel.ts index 5140cc576fc..e3e15f9f19c 100644 --- a/app/actions/remote/channel.ts +++ b/app/actions/remote/channel.ts @@ -45,6 +45,7 @@ import type ChannelModel from '@typings/database/models/servers/channel'; import type {IntlShape} from 'react-intl'; export type MyChannelsRequest = { + teamId?: string; categories?: CategoryWithChannels[]; channels?: Channel[]; memberships?: ChannelMembership[]; @@ -441,7 +442,7 @@ export async function fetchMyChannelsForTeam(serverUrl: string, teamId: string, setTeamLoading(serverUrl, false); } - return {channels, memberships, categories}; + return {channels, memberships, categories, teamId}; } catch (error) { logDebug('error on fetchMyChannelsForTeam', getFullErrorMessage(error)); if (!fetchOnly) { diff --git a/app/actions/remote/entry/common.ts b/app/actions/remote/entry/common.ts index 6a6e56be38a..c9c1ab2142d 100644 --- a/app/actions/remote/entry/common.ts +++ b/app/actions/remote/entry/common.ts @@ -10,7 +10,7 @@ import {fetchPostsForUnreadChannels} from '@actions/remote/post'; import {type MyPreferencesRequest, fetchMyPreferences} from '@actions/remote/preference'; import {fetchRoles} from '@actions/remote/role'; import {fetchConfigAndLicense, fetchDataRetentionPolicy} from '@actions/remote/systems'; -import {fetchMyTeams, fetchTeamsChannelsAndUnreadPosts, handleKickFromTeam, type MyTeamsRequest, updateCanJoinTeams} from '@actions/remote/team'; +import {fetchMyTeams, fetchTeamsChannelsThreadsAndUnreadPosts, handleKickFromTeam, type MyTeamsRequest, updateCanJoinTeams} from '@actions/remote/team'; import {syncTeamThreads} from '@actions/remote/thread'; import {fetchMe, type MyUserRequest, updateAllUsersSince, autoUpdateTimezone} from '@actions/remote/user'; import {General, Preferences, Screens} from '@constants'; @@ -66,9 +66,6 @@ export type EntryResponse = { error: unknown; } -const FETCH_MISSING_DM_TIMEOUT = 2500; -export const FETCH_UNREADS_TIMEOUT = 2500; - export const entry = async (serverUrl: string, teamId?: string, channelId?: string, since = 0): Promise => { const {database} = DatabaseManager.getServerDatabaseAndOperator(serverUrl); const result = entryRest(serverUrl, teamId, channelId, since); @@ -311,8 +308,7 @@ const fetchAlternateTeamData = async ( let initialTeamId = ''; let chData; - for (const teamId of availableTeamIds) { - // eslint-disable-next-line no-await-in-loop + for await (const teamId of availableTeamIds) { chData = await fetchMyChannelsForTeam(serverUrl, teamId, includeDeleted, since, fetchOnly, false, isCRTEnabled); const chError = chData.error; if (isErrorWithStatusCode(chError) && chError.status_code === 403) { @@ -371,49 +367,56 @@ async function entryInitialChannelId(database: Database, requestedChannelId = '' async function restDeferredAppEntryActions( serverUrl: string, since: number, currentUserId: string, currentUserLocale: string, preferences: PreferenceType[] | undefined, config: ClientConfig, license: ClientLicense | undefined, teamData: MyTeamsRequest, chData: MyChannelsRequest | undefined, - initialTeamId?: string, initialChannelId?: string) { - setTimeout(async () => { - if (chData?.channels?.length && chData.memberships?.length) { - // defer fetching posts for unread channels on initial team - fetchPostsForUnreadChannels(serverUrl, chData.channels, chData.memberships, initialChannelId); - } - }, FETCH_UNREADS_TIMEOUT); - - // defer fetch channels and unread posts for other teams - if (teamData.teams?.length && teamData.memberships?.length) { - fetchTeamsChannelsAndUnreadPosts(serverUrl, since, teamData.teams, teamData.memberships, initialTeamId); + initialTeamId?: string, initialChannelId?: string, +) { + const isCRTEnabled = (preferences && processIsCRTEnabled(preferences, config.CollapsedThreads, config.FeatureFlagCollapsedThreads, config.Version)) || false; + const directChannels = chData?.channels?.filter(isDMorGM); + const channelsToFetchProfiles = new Set(directChannels); + + // sidebar DM & GM profiles + if (channelsToFetchProfiles.size) { + const teammateDisplayNameSetting = getTeammateNameDisplaySetting(preferences || [], config.LockTeammateNameDisplay, config.TeammateNameDisplay, license); + fetchMissingDirectChannelsInfo(serverUrl, Array.from(channelsToFetchProfiles), currentUserLocale, teammateDisplayNameSetting, currentUserId); } - if (preferences && processIsCRTEnabled(preferences, config.CollapsedThreads, config.FeatureFlagCollapsedThreads, config.Version)) { - if (initialTeamId) { - await syncTeamThreads(serverUrl, initialTeamId); - } + updateAllUsersSince(serverUrl, since); + updateCanJoinTeams(serverUrl); - if (teamData.teams?.length) { - for await (const team of teamData.teams) { - if (team.id !== initialTeamId) { - // need to await here since GM/DM threads in different teams overlap - await syncTeamThreads(serverUrl, team.id); - } + // defer fetch channels and unread posts for other teams + setTimeout(async () => { + if (chData?.channels?.length && chData.memberships?.length && initialTeamId) { + if (isCRTEnabled && initialTeamId) { + await syncTeamThreads(serverUrl, initialTeamId); } + fetchPostsForUnreadChannels(serverUrl, chData.channels, chData.memberships, initialChannelId); } - } - updateCanJoinTeams(serverUrl); - await updateAllUsersSince(serverUrl, since); + if (teamData.teams?.length && teamData.memberships?.length) { + const teamsOrder = preferences?.find((p) => p.category === Preferences.CATEGORIES.TEAMS_ORDER); + const sortedTeamIds = new Set(teamsOrder?.value.split(',')); + const membershipSet = new Set(teamData.memberships.map((m) => m.team_id)); + const teamMap = new Map(teamData.teams.map((t) => [t.id, t])); + + let myTeams: Team[]; + if (sortedTeamIds.size) { + const mySortedTeams = [...sortedTeamIds]. + filter((id) => membershipSet.has(id) && teamMap.has(id)). + map((id) => teamMap.get(id)!); + const extraTeams = [...membershipSet]. + filter((id) => !sortedTeamIds.has(id) && teamMap.get(id)). + map((id) => teamMap.get(id)!). + sort((a, b) => a.display_name.toLocaleLowerCase().localeCompare(b.display_name.toLocaleLowerCase())); + myTeams = [...mySortedTeams, ...extraTeams]; + } else { + myTeams = teamData.teams. + sort((a, b) => a.display_name.toLocaleLowerCase().localeCompare(b.display_name.toLocaleLowerCase())); + } + fetchTeamsChannelsThreadsAndUnreadPosts(serverUrl, since, myTeams, isCRTEnabled); + } + }); // Fetch groups for current user fetchGroupsForMember(serverUrl, currentUserId); - - // defer sidebar DM & GM profiles - setTimeout(async () => { - const directChannels = chData?.channels?.filter(isDMorGM); - const channelsToFetchProfiles = new Set(directChannels); - if (channelsToFetchProfiles.size) { - const teammateDisplayNameSetting = getTeammateNameDisplaySetting(preferences || [], config.LockTeammateNameDisplay, config.TeammateNameDisplay, license); - fetchMissingDirectChannelsInfo(serverUrl, Array.from(channelsToFetchProfiles), currentUserLocale, teammateDisplayNameSetting, currentUserId); - } - }, FETCH_MISSING_DM_TIMEOUT); } export const setExtraSessionProps = async (serverUrl: string) => { diff --git a/app/actions/remote/post.ts b/app/actions/remote/post.ts index 5ff76826f64..e358a094cb8 100644 --- a/app/actions/remote/post.ts +++ b/app/actions/remote/post.ts @@ -4,6 +4,8 @@ /* eslint-disable max-lines */ +import {chunk} from 'lodash'; + import {markChannelAsUnread, updateLastPostAt} from '@actions/local/channel'; import {addPostAcknowledgement, removePost, removePostAcknowledgement, storePostsForChannel} from '@actions/local/post'; import {addRecentReaction} from '@actions/local/reactions'; @@ -40,6 +42,13 @@ type PostsRequest = { previousPostId?: string; } +export type PostsForChannel = PostsRequest & { + actionType?: string; + authors?: UserProfile[]; + channelId?: string; + error?: unknown; +}; + type PostsObjectsRequest = { error?: unknown; order?: string[]; @@ -275,7 +284,7 @@ export const retryFailedPost = async (serverUrl: string, post: PostModel) => { return {}; }; -export async function fetchPostsForChannel(serverUrl: string, channelId: string, fetchOnly = false) { +export async function fetchPostsForChannel(serverUrl: string, channelId: string, fetchOnly = false): Promise { try { if (!fetchOnly) { EphemeralStore.addLoadingMessagesForChannel(serverUrl, channelId); @@ -312,7 +321,7 @@ export async function fetchPostsForChannel(serverUrl: string, channelId: string, } } - return {posts: data.posts, order: data.order, authors, actionType, previousPostId: data.previousPostId}; + return {posts: data.posts, order: data.order, authors, actionType, previousPostId: data.previousPostId, channelId}; } catch (error) { logDebug('error on fetchPostsForChannel', getFullErrorMessage(error)); return {error}; @@ -323,15 +332,34 @@ export async function fetchPostsForChannel(serverUrl: string, channelId: string, } } -export const fetchPostsForUnreadChannels = async (serverUrl: string, channels: Channel[], memberships: ChannelMembership[], excludeChannelId?: string) => { - const promises = []; +export const fetchPostsForUnreadChannels = async (serverUrl: string, channels: Channel[], memberships: ChannelMembership[], excludeChannelId?: string, fetchOnly = false): Promise => { + const membersMap = new Map(); for (const member of memberships) { - const channel = channels.find((c) => c.id === member.channel_id); - if (channel && !channel.delete_at && (channel.total_msg_count - member.msg_count) > 0 && channel.id !== excludeChannelId) { - promises.push(fetchPostsForChannel(serverUrl, channel.id)); + membersMap.set(member.channel_id, member); + } + + const unreadChannelIds = channels.reduce((result, channel) => { + const member = membersMap.get(channel.id); + if (member && !channel.delete_at && (channel.total_msg_count - member.msg_count) > 0 && channel.id !== excludeChannelId) { + result.push(channel.id); + } + return result; + }, []); + + const postsForChannel: PostsForChannel[] = []; + + // process 10 unread channels at a time + const chunks = chunk(unreadChannelIds, 10); + for await (const channelIds of chunks) { + const promises = []; + for (const channelId of channelIds) { + promises.push(fetchPostsForChannel(serverUrl, channelId, fetchOnly)); } + + const results = await Promise.all(promises); + postsForChannel.push(...results); } - return Promise.all(promises); + return postsForChannel; }; export async function fetchPosts(serverUrl: string, channelId: string, page = 0, perPage = General.POST_CHUNK_SIZE, fetchOnly = false): Promise { diff --git a/app/actions/remote/team.test.ts b/app/actions/remote/team.test.ts index 9f34f7262b0..023fceb9fec 100644 --- a/app/actions/remote/team.test.ts +++ b/app/actions/remote/team.test.ts @@ -18,7 +18,6 @@ import { fetchAllTeams, fetchTeamsForComponent, updateCanJoinTeams, - fetchTeamsChannelsAndUnreadPosts, fetchTeamByName, removeCurrentUserFromTeam, removeUserFromTeam, @@ -26,6 +25,7 @@ import { handleKickFromTeam, getTeamMembersByIds, buildTeamIconUrl, + fetchTeamsChannelsThreadsAndUnreadPosts, } from './team'; import type ServerDataOperator from '@database/operator/server_data_operator'; @@ -81,12 +81,38 @@ const mockClient = { getMyTeams: jest.fn(() => ([{id: teamId, name: 'team1'}])), getMyTeamMembers: jest.fn(() => ([{id: 'userid1-' + teamId, user_id: 'userid1', team_id: teamId, roles: ''}])), getTeams: jest.fn(() => ([{id: teamId, name: 'team1'}])), - getMyChannels: jest.fn((id: string) => ([{id: channelId, name: 'channel1', creatorId: user.id, team_id: id}])), - getMyChannelMembers: jest.fn(() => ([{id: user.id + '-' + channelId, user_id: user.id, channel_id: channelId, roles: ''}])), - getCategories: jest.fn((userId: string, id: string) => ({categories: [{id: 'categoryid', channel_id: [channelId], team_id: id}], order: ['categoryid']})), + getMyChannels: jest.fn((id: string) => ([{id: channelId, name: 'channel1', creatorId: user.id, team_id: id, total_msg_count: 1}])), + getMyChannelMembers: jest.fn(() => ([{id: user.id + '-' + channelId, user_id: user.id, channel_id: channelId, roles: '', msg_count: 0}])), + getCategories: jest.fn((userId: string, id: string) => ({categories: [{id: 'categoryid', channel_ids: [channelId], team_id: id}], order: ['categoryid']})), getTeamByName: jest.fn((name: string) => ({id: teamId, name})), removeFromTeam: jest.fn(), getTeamMembersByIds: jest.fn((id: string, userIds: string[]) => (userIds.map((userId) => ({id: userId + '-' + id, user_id: userId, team_id: id, roles: ''})))), + getPosts: jest.fn(() => ({ + order: ['yocj9xgkh78exk1uhx9yny1zxy', 'ad6yoisgh385fy7rkph49zpfqa', 'o5qqa4ntdigp7rbnf75f8hgeaw'], + posts: [{ + channel_id: channelId, + create_at: 1726107604522, + delete_at: 0, + edit_at: 0, + hashtags: '', + id: 'yocj9xgkh78exk1uhx9yny1zxy', + is_pinned: false, + last_reply_at: 0, + message: 'Message ead863', + metadata: {}, + original_id: '', + participants: null, + pending_post_id: '', + props: {}, + remote_id: '', + reply_count: 0, + root_id: '', + type: '', + update_at: 1726107604522, + user_id: 'k479wsypafgjddz8cerqx4m1ha', + }], + previousPostId: '', + })), }; beforeAll(() => { @@ -245,16 +271,25 @@ describe('teams', () => { expect(result).toBeDefined(); }); - it('fetchTeamsChannelsAndUnreadPosts - handle not found database', async () => { - const result = await fetchTeamsChannelsAndUnreadPosts('foo', 0, [], []) as {error: unknown}; + it('fetchTeamsChannelsThreadsAndUnreadPosts - handle not found database', async () => { + const result = await fetchTeamsChannelsThreadsAndUnreadPosts('foo', 0, []) as {error: unknown}; expect(result?.error).toBeDefined(); }); - it('fetchTeamsChannelsAndUnreadPosts - base case', async () => { - const result = await fetchTeamsChannelsAndUnreadPosts(serverUrl, 0, [team], [{team_id: teamId, user_id: 'userid1'} as TeamMembership]); + it('fetchTeamsChannelsThreadsAndUnreadPosts - base case', async () => { + const result = await fetchTeamsChannelsThreadsAndUnreadPosts( + serverUrl, 0, + [team], true, false); expect(result).toBeDefined(); }); + it('fetchTeamsChannelsThreadsAndUnreadPosts - fetch only case', async () => { + const result = await fetchTeamsChannelsThreadsAndUnreadPosts( + serverUrl, 0, + [team], true, true); + expect(result.models).toBeDefined(); + }); + it('fetchTeamByName - handle not found database', async () => { const result = await fetchTeamByName('foo', '') as {error: unknown}; expect(result?.error).toBeDefined(); diff --git a/app/actions/remote/team.ts b/app/actions/remote/team.ts index 3d62a0231ee..364c35a40af 100644 --- a/app/actions/remote/team.ts +++ b/app/actions/remote/team.ts @@ -1,12 +1,17 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. +import {chunk} from 'lodash'; import {DeviceEventEmitter} from 'react-native'; +import {storeCategories} from '@actions/local/category'; +import {storeMyChannelsForTeam} from '@actions/local/channel'; +import {storePostsForChannel} from '@actions/local/post'; import {removeUserFromTeam as localRemoveUserFromTeam} from '@actions/local/team'; import {PER_PAGE_DEFAULT} from '@client/rest/constants'; import {Events} from '@constants'; import DatabaseManager from '@database/manager'; +import {removeDuplicatesModels} from '@helpers/database'; import NetworkManager from '@managers/network_manager'; import {getActiveServerUrl} from '@queries/app/servers'; import {prepareCategoriesAndCategoriesChannels} from '@queries/servers/categories'; @@ -22,9 +27,10 @@ import {logDebug} from '@utils/log'; import {fetchMyChannelsForTeam, switchToChannelById} from './channel'; import {fetchGroupsForTeamIfConstrained} from './groups'; -import {fetchPostsForChannel, fetchPostsForUnreadChannels} from './post'; +import {fetchPostsForChannel, fetchPostsForUnreadChannels, type PostsForChannel} from './post'; import {fetchRolesIfNeeded} from './role'; import {forceLogoutIfNecessary} from './session'; +import {syncThreadsIfNeeded} from './thread'; import type {Client} from '@client/rest'; import type {Model} from '@nozbe/watermelondb'; @@ -312,24 +318,85 @@ export const updateCanJoinTeams = async (serverUrl: string) => { } }; -export const fetchTeamsChannelsAndUnreadPosts = async (serverUrl: string, since: number, teams: Team[], memberships: TeamMembership[], excludeTeamId?: string) => { - const database = DatabaseManager.serverDatabases[serverUrl]?.database; - if (!database) { - return {error: `${serverUrl} database not found`}; - } +export const fetchTeamsChannelsThreadsAndUnreadPosts = async ( + serverUrl: string, since: number, teams: Team[], + isCRTEnabled?: boolean, fetchOnly = false, +) => { + try { + const {operator} = DatabaseManager.getServerDatabaseAndOperator(serverUrl); + const result: Model[] = []; + + // process up to 15 teams at a time + const chunks = chunk(teams, 15); + for await (const myTeams of chunks) { + const promises = []; + for (const team of myTeams) { + promises.push(fetchMyChannelsForTeam(serverUrl, team.id, true, since, true, true, isCRTEnabled)); + } + + const results = await Promise.all(promises); + const models: Model[] = []; + const channels: Channel[] = []; + const members: ChannelMembership[] = []; + + for await (const r of results) { + if (!r.error && r.teamId && r.categories && r.channels && r.memberships) { + const {models: chModels} = await storeMyChannelsForTeam(serverUrl, r.teamId, r.channels, r.memberships, true, isCRTEnabled); + const {models: catModels} = await storeCategories(serverUrl, r.categories, true, true); // Re-sync + if (chModels?.length) { + models.push(...chModels); + } - const membershipSet = new Set(memberships.map((m) => m.team_id)); - const myTeams = teams.filter((t) => membershipSet.has(t.id) && t.id !== excludeTeamId); + if (catModels?.length) { + models.push(...catModels); + } - for await (const team of myTeams) { - const {channels, memberships: members} = await fetchMyChannelsForTeam(serverUrl, team.id, true, since, false, true); + channels.push(...r.channels); + members.push(...r.memberships); + } + } - if (channels?.length && members?.length) { - fetchPostsForUnreadChannels(serverUrl, channels, members); + const unreadPromise = fetchPostsForUnreadChannels(serverUrl, channels, members, undefined, true); + const threadsPromise = syncThreadsIfNeeded(serverUrl, isCRTEnabled ?? false, myTeams, true); + const postPromises: [Promise, Promise<{models?: Model[]; error?: unknown}>] = [ + unreadPromise, + threadsPromise, + ]; + + const [unreadPosts, threads] = await Promise.all(postPromises); + + for await (const data of unreadPosts) { + if (data.channelId && data.posts?.length && data.order?.length && data.actionType) { + const {models: prepared} = await storePostsForChannel( + serverUrl, data.channelId, + data.posts, data.order, data.previousPostId ?? '', + data.actionType, data.authors || [], true, + ); + + if (prepared?.length) { + models.push(...prepared); + } + } + } + + if (threads.models) { + models.push(...threads.models); + } + + if (!fetchOnly && models.length) { + await operator.batchRecords(removeDuplicatesModels(models), 'fetchTeamsChannelsThreadsAndUnreadPosts'); + } + + if (models.length) { + result.push(...models); + } } - } - return {}; + return {error: undefined, models: result}; + } catch (error) { + logDebug('error on fetchTeamsChannelsThreadsAndUnreadPosts', getFullErrorMessage(error)); + return {error}; + } }; export async function fetchTeamByName(serverUrl: string, teamName: string, fetchOnly = false) { diff --git a/app/actions/remote/thread.test.ts b/app/actions/remote/thread.test.ts index 0352bb8b39d..293da7cc77c 100644 --- a/app/actions/remote/thread.test.ts +++ b/app/actions/remote/thread.test.ts @@ -19,6 +19,7 @@ import { syncTeamThreads, loadEarlierThreads, fetchAndSwitchToThread, + syncThreadsIfNeeded, } from './thread'; import type ServerDataOperator from '@database/operator/server_data_operator'; @@ -150,7 +151,7 @@ describe('get threads', () => { expect(result).toBeDefined(); expect(result.error).toBeFalsy(); expect(result.models).toBeDefined(); - expect(result.models?.length).toBe(5); // 1 thread, 1 thread participant, 1 read thread in team, 1 unread thread in team, 1 team thread sync + expect(result.models?.length).toBe(4); // 1 thread, 1 thread participant, 1 latest thread in team, 1 team thread sync (the unread thread is actually the latest so it the duplicate is removed) // Sync again after first sync const result2 = await syncTeamThreads(serverUrl, team.id); @@ -160,6 +161,23 @@ describe('get threads', () => { expect(result2.models?.length).toBe(1); // 1 team thread sync }); + it('syncThreadsIfNeeded - handle error', async () => { + const result = await syncThreadsIfNeeded('foo', true, []); + expect(result).toBeDefined(); + expect(result.error).toBeTruthy(); + }); + + it('syncThreadsIfNeeded - base case', async () => { + await operator.handleSystem({systems: [{id: SYSTEM_IDENTIFIERS.CURRENT_USER_ID, value: user1.id}], prepareRecordsOnly: false}); + await operator.handleUsers({users: [user1], prepareRecordsOnly: false}); + + const result = await syncThreadsIfNeeded(serverUrl, true, [team]); + expect(result).toBeDefined(); + expect(result.error).toBeFalsy(); + expect(result.models).toBeDefined(); + expect(result.models?.length).toBe(4); // 1 thread, 1 thread participant, 1 latest thread in team, 1 team thread sync (the unread thread is actually the latest so it the duplicate is removed) + }); + it('loadEarlierThreads - handle error', async () => { const result = await loadEarlierThreads('foo', '', ''); expect(result).toBeDefined(); diff --git a/app/actions/remote/thread.ts b/app/actions/remote/thread.ts index a72142b93e1..a6b6534c187 100644 --- a/app/actions/remote/thread.ts +++ b/app/actions/remote/thread.ts @@ -5,6 +5,7 @@ import {markTeamThreadsAsRead, markThreadAsViewed, processReceivedThreads, switc import {fetchPostThread} from '@actions/remote/post'; import {General} from '@constants'; import DatabaseManager from '@database/manager'; +import {removeDuplicatesModels} from '@helpers/database'; import PushNotifications from '@init/push_notifications'; import AppsManager from '@managers/apps_manager'; import NetworkManager from '@managers/network_manager'; @@ -13,7 +14,8 @@ import {getConfigValue, getCurrentChannelId, getCurrentTeamId} from '@queries/se import {getIsCRTEnabled, getThreadById, getTeamThreadsSyncData} from '@queries/servers/thread'; import {getCurrentUser} from '@queries/servers/user'; import {getFullErrorMessage} from '@utils/errors'; -import {logDebug} from '@utils/log'; +import {isMinimumServerVersion} from '@utils/helpers'; +import {logDebug, logError} from '@utils/log'; import {showThreadFollowingSnackbar} from '@utils/snack_bar'; import {getThreadsListEdges} from '@utils/thread'; @@ -30,6 +32,7 @@ type FetchThreadsOptions = { unread?: boolean; since?: number; totalsOnly?: boolean; + excludeDirect?: boolean; }; enum Direction { @@ -238,10 +241,10 @@ export const fetchThreads = async ( let currentPage = 0; const fetchThreadsFunc = async (opts: FetchThreadsOptions) => { - const {before, after, perPage = General.CRT_CHUNK_SIZE, deleted, unread, since} = opts; + const {before, after, perPage = General.CRT_CHUNK_SIZE, deleted, unread, since, excludeDirect = false} = opts; currentPage++; - const {threads} = await client.getThreads(currentUser.id, teamId, before, after, perPage, deleted, unread, since, false, version); + const {threads} = await client.getThreads(currentUser.id, teamId, before, after, perPage, deleted, unread, since, false, version, excludeDirect); if (threads.length) { // Mark all fetched threads as following for (const thread of threads) { @@ -277,7 +280,50 @@ export const fetchThreads = async ( return {error: false, threads: threadsData}; }; -export const syncTeamThreads = async (serverUrl: string, teamId: string, prepareRecordsOnly = false) => { +export const syncThreadsIfNeeded = async (serverUrl: string, isCRTEnabled: boolean, teams?: Team[], fetchOnly = false) => { + try { + if (!isCRTEnabled) { + return {models: []}; + } + + const {database, operator} = DatabaseManager.getServerDatabaseAndOperator(serverUrl); + const promises = []; + const models: Model[][] = []; + + // this is to keep backwards compatibility with servers that send the + // threads for DM / GM regardless for every team + const version = await getConfigValue(database, 'Version'); + const hasThreadExclusions = isMinimumServerVersion(version, 10, 2, 0); + + if (teams?.length) { + for (const team of teams) { + promises.push(syncTeamThreads(serverUrl, team.id, true, true)); + } + } + + if (promises.length) { + const results = await Promise.all(promises); + for (const r of results) { + if (r.models?.length) { + models.push(r.models); + } + } + } + + const flat = models.flat(); + if (!fetchOnly && flat.length) { + const uniqueArray = hasThreadExclusions ? flat : removeDuplicatesModels(flat); + await operator.batchRecords(uniqueArray, 'syncThreadsIfNeeded'); + } + + return {models: flat}; + } catch (error) { + logError('syncThreadsIfNeeded: Error', error); + return {error, models: undefined}; + } +}; + +export const syncTeamThreads = async (serverUrl: string, teamId: string, excludeDirect = false, fetchOnly = false) => { try { const {database, operator} = DatabaseManager.getServerDatabaseAndOperator(serverUrl); const syncData = await getTeamThreadsSyncData(database, teamId); @@ -299,13 +345,13 @@ export const syncTeamThreads = async (serverUrl: string, teamId: string, prepare fetchThreads( serverUrl, teamId, - {unread: true}, + {unread: true, excludeDirect}, Direction.Down, ), fetchThreads( serverUrl, teamId, - {}, + {excludeDirect}, undefined, 1, ), @@ -313,6 +359,9 @@ export const syncTeamThreads = async (serverUrl: string, teamId: string, prepare if (allUnreadThreads.error || latestThreads.error) { return {error: allUnreadThreads.error || latestThreads.error}; } + + const dedupe = new Set(latestThreads.threads?.map((t) => t.id)); + if (latestThreads.threads?.length) { // We are fetching the threads for the first time. We get "latest" and "earliest" values. const {earliestThread, latestThread} = getThreadsListEdges(latestThreads.threads); @@ -322,13 +371,14 @@ export const syncTeamThreads = async (serverUrl: string, teamId: string, prepare threads.push(...latestThreads.threads); } if (allUnreadThreads.threads?.length) { - threads.push(...allUnreadThreads.threads); + const unread = allUnreadThreads.threads.filter((u) => !dedupe.has(u.id)); + threads.push(...unread); } } else { const allNewThreads = await fetchThreads( serverUrl, teamId, - {deleted: true, since: syncData.latest + 1}, + {deleted: true, since: syncData.latest + 1, excludeDirect}, ); if (allNewThreads.error) { return {error: allNewThreads.error}; @@ -361,7 +411,7 @@ export const syncTeamThreads = async (serverUrl: string, teamId: string, prepare } } - if (!prepareRecordsOnly && models?.length) { + if (!fetchOnly && models?.length) { try { await operator.batchRecords(models, 'syncTeamThreads'); } catch (err) { @@ -372,7 +422,6 @@ export const syncTeamThreads = async (serverUrl: string, teamId: string, prepare } } } - return {error: false, models}; } catch (error) { return {error}; diff --git a/app/actions/websocket/channel.ts b/app/actions/websocket/channel.ts index 91a32e54834..c4934b0234f 100644 --- a/app/actions/websocket/channel.ts +++ b/app/actions/websocket/channel.ts @@ -314,11 +314,11 @@ export async function handleUserAddedToChannelEvent(serverUrl: string, msg: any) } const {posts, order, authors, actionType, previousPostId} = await fetchPostsForChannel(serverUrl, channelId, true); - if (posts?.length && order?.length) { + if (posts?.length && order?.length && actionType) { const {models: prepared} = await storePostsForChannel( serverUrl, channelId, posts, order, previousPostId ?? '', - actionType, authors, true, + actionType, authors || [], true, ); if (prepared?.length) { diff --git a/app/client/rest/threads.ts b/app/client/rest/threads.ts index c071c8fa166..5a5c79cb1ea 100644 --- a/app/client/rest/threads.ts +++ b/app/client/rest/threads.ts @@ -8,7 +8,7 @@ import {PER_PAGE_DEFAULT} from './constants'; import type ClientBase from './base'; export interface ClientThreadsMix { - getThreads: (userId: string, teamId: string, before?: string, after?: string, pageSize?: number, deleted?: boolean, unread?: boolean, since?: number, totalsOnly?: boolean, serverVersion?: string) => Promise; + getThreads: (userId: string, teamId: string, before?: string, after?: string, pageSize?: number, deleted?: boolean, unread?: boolean, since?: number, totalsOnly?: boolean, serverVersion?: string, excludeDirect?: boolean) => Promise; getThread: (userId: string, teamId: string, threadId: string, extended?: boolean) => Promise; markThreadAsRead: (userId: string, teamId: string, threadId: string, timestamp: number) => Promise; markThreadAsUnread: (userId: string, teamId: string, threadId: string, postId: string) => Promise; @@ -17,7 +17,7 @@ export interface ClientThreadsMix { } const ClientThreads = >(superclass: TBase) => class extends superclass { - getThreads = async (userId: string, teamId: string, before = '', after = '', pageSize = PER_PAGE_DEFAULT, deleted = false, unread = false, since = 0, totalsOnly = false, serverVersion = '') => { + getThreads = async (userId: string, teamId: string, before = '', after = '', pageSize = PER_PAGE_DEFAULT, deleted = false, unread = false, since = 0, totalsOnly = false, serverVersion = '', excludeDirect = false) => { const queryStringObj: Record = { extended: 'true', before, @@ -26,6 +26,7 @@ const ClientThreads = >(superclass: TBase) unread, since, totalsOnly, + excludeDirect, }; if (serverVersion && isMinimumServerVersion(serverVersion, 6, 0)) { queryStringObj.per_page = pageSize; diff --git a/app/components/team_sidebar/team_list/index.ts b/app/components/team_sidebar/team_list/index.ts index d30b8c24b7c..117c9c9c8db 100644 --- a/app/components/team_sidebar/team_list/index.ts +++ b/app/components/team_sidebar/team_list/index.ts @@ -16,6 +16,11 @@ import TeamList from './team_list'; import type {WithDatabaseArgs} from '@typings/database/database'; import type MyTeamModel from '@typings/database/models/servers/my_team'; +interface TeamWithLowerName { + myTeam: MyTeamModel; + lowerName?: string; +} + const withTeams = withObservables([], ({database}: WithDatabaseArgs) => { const myTeams = queryMyTeams(database).observe(); const teamIds = queryJoinedTeams(database).observe().pipe( @@ -26,26 +31,41 @@ const withTeams = withObservables([], ({database}: WithDatabaseArgs) => { switchMap((p) => (p.length ? of$(p[0].value.split(',')) : of$([]))), ); const myOrderedTeams = combineLatest([myTeams, order, teamIds]).pipe( - map(([ts, o, tids]) => { - let ids: string[] = o; - if (!o.length) { - ids = tids. - sort((a, b) => a.displayName.toLocaleLowerCase().localeCompare(b.displayName.toLocaleLowerCase())). - map((t) => t.id); + map(([memberships, o, teams]) => { + const sortedTeamIds = new Set(o); + const membershipMap = new Map(memberships.map((m) => [m.id, m])); + + if (sortedTeamIds.size) { + const mySortedTeams = [...sortedTeamIds]. + filter((id) => membershipMap.has(id)). + map((id) => membershipMap.get(id)!); + + const extraTeams = teams. + filter((t) => !sortedTeamIds.has(t.id) && membershipMap.has(t.id)). + map((t) => ({ + myTeam: membershipMap.get(t.id)!, + lowerName: t.displayName.toLocaleLowerCase(), + } as TeamWithLowerName)). + sort((a, b) => a.lowerName!.localeCompare(b.lowerName!)). + map((t) => { + delete t.lowerName; + return t; + }); + + return [...mySortedTeams, ...extraTeams]; } - const teamMap = ts.reduce<{[x: string]: MyTeamModel}>((result, team) => { - result[team.id] = team; - return result; - }, {}); - - return ids.reduce((result, id) => { - const t = teamMap[id]; - if (t) { - result.push(t); - } - return result; - }, []); + return teams. + filter((t) => membershipMap.has(t.id)). + map((t) => ({ + myTeam: membershipMap.get(t.id)!, + lowerName: t.displayName.toLocaleLowerCase(), + } as TeamWithLowerName)). + sort((a, b) => a.lowerName!.localeCompare(b.lowerName!)). + map((t) => { + delete t.lowerName; + return t.myTeam; + }); }), ); return { diff --git a/app/database/operator/server_data_operator/transformers/thread.ts b/app/database/operator/server_data_operator/transformers/thread.ts index 27c9c443410..19d170504d8 100644 --- a/app/database/operator/server_data_operator/transformers/thread.ts +++ b/app/database/operator/server_data_operator/transformers/thread.ts @@ -63,9 +63,12 @@ export const transformThreadRecord = ({action, database, value}: TransformerArgs */ export const transformThreadParticipantRecord = ({action, database, value}: TransformerArgs): Promise => { const raw = value.raw as ThreadParticipant; + const record = value.record as ThreadParticipantModel; + const isCreateAction = action === OperationType.CREATE; // id of participant comes from server response const fieldsMapper = (participant: ThreadParticipantModel) => { + participant._raw.id = isCreateAction ? `${raw.thread_id}-${raw.id}` : record.id; participant.threadId = raw.thread_id; participant.userId = raw.id; }; @@ -81,8 +84,11 @@ export const transformThreadParticipantRecord = ({action, database, value}: Tran export const transformThreadInTeamRecord = ({action, database, value}: TransformerArgs): Promise => { const raw = value.raw as ThreadInTeam; + const record = value.record as ThreadInTeamModel; + const isCreateAction = action === OperationType.CREATE; const fieldsMapper = (threadInTeam: ThreadInTeamModel) => { + threadInTeam._raw.id = isCreateAction ? `${raw.thread_id}-${raw.team_id}` : record.id; threadInTeam.threadId = raw.thread_id; threadInTeam.teamId = raw.team_id; }; diff --git a/app/helpers/database/index.ts b/app/helpers/database/index.ts index 4952bee4d35..0fd7082d14d 100644 --- a/app/helpers/database/index.ts +++ b/app/helpers/database/index.ts @@ -5,6 +5,7 @@ import xRegExp from 'xregexp'; import {General} from '@constants'; +import type {Model} from '@nozbe/watermelondb'; import type ChannelModel from '@typings/database/models/servers/channel'; import type MyChannelModel from '@typings/database/models/servers/my_channel'; @@ -91,3 +92,15 @@ export const filterAndSortMyChannels = ([myChannels, channels, notifyProps]: Fil // Matches letters from any alphabet and numbers const sqliteLikeStringRegex = xRegExp('[^\\p{L}\\p{Nd}]', 'g'); export const sanitizeLikeString = (value: string) => value.replace(sqliteLikeStringRegex, '_'); + +export function removeDuplicatesModels(array: Model[]) { + const seen = new Set(); + return array.filter((item) => { + const key = `${item.collection.table}-${item.id}`; + if (seen.has(key)) { + return false; + } + seen.add(key); + return true; + }); +} diff --git a/app/managers/network_manager.ts b/app/managers/network_manager.ts index 4d6afb81d72..b5bd94e1b4a 100644 --- a/app/managers/network_manager.ts +++ b/app/managers/network_manager.ts @@ -60,7 +60,7 @@ class NetworkManager { sessionConfiguration: { allowsCellularAccess: true, waitsForConnectivity: false, - httpMaximumConnectionsPerHost: 10, + httpMaximumConnectionsPerHost: 100, cancelRequestsOnUnauthorized: true, }, retryPolicyConfiguration: { diff --git a/ios/Podfile.lock b/ios/Podfile.lock index 2ea5bec7945..72a447e3af0 100644 --- a/ios/Podfile.lock +++ b/ios/Podfile.lock @@ -1162,7 +1162,7 @@ PODS: - Yoga - react-native-netinfo (11.3.2): - React-Core - - react-native-network-client (1.7.2): + - react-native-network-client (1.7.3): - Alamofire (~> 5.9.1) - DoubleConversion - glog @@ -2043,7 +2043,7 @@ SPEC CHECKSUMS: react-native-emm: ecab44d78fb1cc7d7b7901914f48fb6309c46ff2 react-native-image-picker: c3afe5472ef870d98a4b28415fc0b928161ee5f7 react-native-netinfo: 076df4f9b07f6670acf4ce9a75aac8d34c2e2ccc - react-native-network-client: 5173c47230b5f497cdef469cba8e6e1b3df687eb + react-native-network-client: 82ef7c000c5fd3bfd1ad9e78cd920af33040f9f3 react-native-notifications: 4601a5a8db4ced6ae7cfc43b44d35fe437ac50c4 react-native-paste-input: 011a9916ef3acf809a7da122847c61ca0f93a60e react-native-performance: ff93f8af3b2ee9519fd7879896aa9b8b8272691d diff --git a/libraries/@mattermost/rnutils/ios/RNUtilsWrapper.swift b/libraries/@mattermost/rnutils/ios/RNUtilsWrapper.swift index c9f800841c6..ecc50e4ff21 100644 --- a/libraries/@mattermost/rnutils/ios/RNUtilsWrapper.swift +++ b/libraries/@mattermost/rnutils/ios/RNUtilsWrapper.swift @@ -6,7 +6,7 @@ import React @objc private var hasRegisteredLoad = false deinit { - DispatchQueue.main.async { + DispatchQueue.main.sync { guard let w = UIApplication.shared.delegate?.window, let window = w else { return } window.removeObserver(self, forKeyPath: "frame") } diff --git a/package-lock.json b/package-lock.json index da6867c432f..7256e058e52 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "mattermost-mobile", - "version": "2.21.0", + "version": "2.22.0", "hasInstallScript": true, "license": "Apache 2.0", "dependencies": { @@ -22,7 +22,7 @@ "@mattermost/hardware-keyboard": "file:./libraries/@mattermost/hardware-keyboard", "@mattermost/keyboard-tracker": "file:./libraries/@mattermost/keyboard-tracker", "@mattermost/react-native-emm": "1.5.0", - "@mattermost/react-native-network-client": "1.7.2", + "@mattermost/react-native-network-client": "1.7.3", "@mattermost/react-native-paste-input": "0.8.0", "@mattermost/react-native-turbo-log": "0.4.0", "@mattermost/rnshare": "file:./libraries/@mattermost/rnshare", @@ -5908,9 +5908,9 @@ } }, "node_modules/@mattermost/react-native-network-client": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/@mattermost/react-native-network-client/-/react-native-network-client-1.7.2.tgz", - "integrity": "sha512-UaFhUaQkmb6Hq2LOyutiwL0HeMVHmhBBpgojLi83zXXXB5OgNWDgqbDBo6Y986c5TwhCy1/LxENdHBc+mI4RZA==", + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/@mattermost/react-native-network-client/-/react-native-network-client-1.7.3.tgz", + "integrity": "sha512-gJSQ4Prf5jcbPlxVylhEtBGafSBYZSat+T21LIDHksGOfeY+jvPL3p8dS+cq+0D1AOHQ3tHNBkZ7RaY6IdUadw==", "license": "MIT", "dependencies": { "validator": "13.12.0", diff --git a/package.json b/package.json index 8811138cef1..c8fd07de064 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "@mattermost/hardware-keyboard": "file:./libraries/@mattermost/hardware-keyboard", "@mattermost/keyboard-tracker": "file:./libraries/@mattermost/keyboard-tracker", "@mattermost/react-native-emm": "1.5.0", - "@mattermost/react-native-network-client": "1.7.2", + "@mattermost/react-native-network-client": "1.7.3", "@mattermost/react-native-paste-input": "0.8.0", "@mattermost/react-native-turbo-log": "0.4.0", "@mattermost/rnshare": "file:./libraries/@mattermost/rnshare",