Skip to content

Commit

Permalink
uniques full sync
Browse files Browse the repository at this point in the history
  • Loading branch information
valentunn committed Feb 25, 2022
1 parent d9dc3ea commit 7faf275
Show file tree
Hide file tree
Showing 19 changed files with 234 additions and 32 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package io.novafoundation.nova

import android.content.Context
import androidx.test.core.app.ApplicationProvider
import io.novafoundation.nova.common.di.FeatureUtils
import io.novafoundation.nova.feature_account_api.di.AccountFeatureApi
import io.novafoundation.nova.feature_nft_api.NftFeatureApi
import io.novafoundation.nova.feature_nft_api.data.model.Nft
import io.novafoundation.nova.feature_nft_api.data.model.isFullySynced
import io.novafoundation.nova.runtime.di.RuntimeApi
import io.novafoundation.nova.runtime.di.RuntimeComponent
import io.novafoundation.nova.runtime.multiNetwork.connection.ChainConnection
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.flow.takeWhile
import kotlinx.coroutines.runBlocking
import org.junit.Test

class NftFullSyncIntegrationTest {

private val nftApi = FeatureUtils.getFeature<NftFeatureApi>(
ApplicationProvider.getApplicationContext<Context>(),
NftFeatureApi::class.java
)

private val accountApi = FeatureUtils.getFeature<AccountFeatureApi>(
ApplicationProvider.getApplicationContext<Context>(),
AccountFeatureApi::class.java
)

private val runtimeApi = FeatureUtils.getFeature<RuntimeComponent>(
ApplicationProvider.getApplicationContext<Context>(),
RuntimeApi::class.java
)

private val externalRequirementFlow = runtimeApi.externalRequirementFlow()

@Test
fun testUniquesIntegration(): Unit = runBlocking {
externalRequirementFlow.emit(ChainConnection.ExternalRequirement.ALLOWED)

val metaAccount = accountApi.accountUseCase().getSelectedMetaAccount()

val nftRepository = nftApi.nftRepository

nftRepository.initialNftSync(metaAccount)

nftRepository.allNftFlow(metaAccount)
.map { nfts -> nfts.filter { it.type is Nft.Type.Uniques && !it.isFullySynced } }
.takeWhile { it.isNotEmpty() }
.onEach { unsyncedNfts ->
unsyncedNfts.forEach { nftRepository.fullNftSync(it) }
}.launchIn(this)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,13 @@ abstract class NftDao {
protected abstract suspend fun insertNfts(nfts: List<NftLocal>)

@Update
abstract suspend fun updateNft(nft: NftLocal)
protected abstract suspend fun updateNft(nft: NftLocal)

@Query("SELECT * FROM nfts WHERE identifier = :nftIdentifier")
protected abstract suspend fun getNft(nftIdentifier: String): NftLocal

@Query("UPDATE nfts SET wholeDetailsLoaded = 1 WHERE identifier = :nftIdentifier")
abstract suspend fun markFullSynced(nftIdentifier: String)

@Transaction
open suspend fun insertNftsDiff(nftType: NftLocal.Type, metaId: Long, newNfts: List<NftLocal>) {
Expand All @@ -38,4 +44,13 @@ abstract class NftDao {
deleteNfts(diff.removed)
insertNfts(diff.newOrUpdated)
}

@Transaction
open suspend fun updateNft(nftIdentifier: String, update: (NftLocal) -> NftLocal) {
val nft = getNft(nftIdentifier)

val updated = update(nft)

updateNft(updated)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import io.novafoundation.nova.common.utils.optionalContentEquals
import java.math.BigInteger

@Entity(tableName = "nfts")
class NftLocal(
data class NftLocal(
@PrimaryKey
override val identifier: String,
@ColumnInfo(index = true)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
android:layout_height="match_parent"
tools:background="@drawable/drawable_background_image">


<androidx.recyclerview.widget.RecyclerView
android:id="@+id/balanceListAssets"
android:layout_width="match_parent"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import jp.co.soramitsu.fearless_utils.runtime.AccountId
import java.math.BigInteger

class Nft(
val identifier: String,
val chain: Chain,
val owner: AccountId,
val metadataRaw: ByteArray?,
Expand Down Expand Up @@ -32,7 +33,7 @@ class Nft(
sealed class Issuance {
class Unlimited(val edition: String) : Issuance()

class Limited(val max: Int, val edition: Int): Issuance()
class Limited(val max: Int, val edition: Int) : Issuance()
}

sealed class Type {
Expand All @@ -44,3 +45,6 @@ class Nft(
class Rmrk2(val collectionId: String) : Type()
}
}

val Nft.isFullySynced
get() = details is Nft.Details.Loaded
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ fun mapNftLocalToNft(
}

return Nft(
identifier = nftLocal.identifier,
chain = chain,
owner = metaAccount.accountIdIn(chain)!!,
metadataRaw = nftLocal.metadata,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,41 +1,46 @@
package io.novafoundation.nova.feature_nft_impl.data.network.distributed

enum class FileStorage(val protocol: String, val defaultHttpsGateway: String?) {
IPFS("ipfs", "https://cloudflare-ipfs.com"),
HTTPS("https", null),
HTTP("http", null);
enum class FileStorage(val prefix: String, val defaultHttpsGateway: String?) {
IPFS("ipfs://ipfs/", "https://rmrk.mypinata.cloud/ipfs/"),
HTTPS("https://", null),
HTTP("http://", null);

init {
require(!defaultHttpsGateway.orEmpty().endsWith("/")) {
"Gateway should not end with '/' separator"
}
require(!protocol.endsWith("://")) {
"Protocol should not end with '://' separator"
}
validateHttpsGateway(defaultHttpsGateway)
}
}

val FileStorage.protocolPrefix
get() = "$protocol://"
private fun validateHttpsGateway(gateway: String?) {
require(gateway == null || gateway.endsWith("/")) {
"Gateway should end with '/' separator"
}
}

object FileStorageAdapter {

fun String.adoptFileStorageLinkToHttps(
customGateways: Map<FileStorage, String> = emptyMap(),
noProtocolStorage: FileStorage = FileStorage.IPFS
) = adaptToHttps(this, customGateways, noProtocolStorage)

fun adaptToHttps(
distributedStorageLink: String,
customGateways: Map<FileStorage, String> = emptyMap()
): String? {
customGateways: Map<FileStorage, String> = emptyMap(),
noProtocolStorage: FileStorage = FileStorage.IPFS
): String {
val distributedStorage = FileStorage.values().firstOrNull { storage ->
distributedStorageLink.pointsTo(storage)
} ?: return null
} ?: noProtocolStorage

val gateway = customGateways[distributedStorage] ?: distributedStorage.defaultHttpsGateway
?: return distributedStorageLink

val path = distributedStorageLink.removePrefix(distributedStorage.protocolPrefix)
validateHttpsGateway(gateway)

val path = distributedStorageLink.removePrefix(distributedStorage.prefix)

return "$gateway/$path"
return "$gateway$path"
}

private fun String.pointsTo(fileStorage: FileStorage) = startsWith(fileStorage.protocolPrefix)
private fun String.pointsTo(fileStorage: FileStorage) = startsWith(fileStorage.prefix)
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import io.novafoundation.nova.feature_account_api.domain.model.MetaAccount
import io.novafoundation.nova.feature_nft_api.data.model.Nft
import io.novafoundation.nova.feature_nft_api.data.repository.NftRepository
import io.novafoundation.nova.feature_nft_impl.data.mappers.mapNftLocalToNft
import io.novafoundation.nova.feature_nft_impl.data.source.JobOrchestrator
import io.novafoundation.nova.feature_nft_impl.data.source.NftProvidersRegistry
import io.novafoundation.nova.runtime.multiNetwork.ChainRegistry
import kotlinx.coroutines.Dispatchers
Expand All @@ -21,6 +22,7 @@ private const val NFT_TAG = "NFT"
class NftRepositoryImpl(
private val nftProvidersRegistry: NftProvidersRegistry,
private val chainRegistry: ChainRegistry,
private val jobOrchestrator: JobOrchestrator,
private val nftDao: NftDao,
) : NftRepository {

Expand Down Expand Up @@ -55,7 +57,13 @@ class NftRepositoryImpl(
syncJobs.joinAll()
}

override suspend fun fullNftSync(nft: Nft) {
TODO("Not yet implemented")
override suspend fun fullNftSync(nft: Nft) = withContext(Dispatchers.IO) {
jobOrchestrator.runUniqueJob(nft.identifier) {
runCatching {
nftProvidersRegistry.get(nft).nftFullSync(nft)
}.onFailure {
Log.e(NFT_TAG, "Failed to fully sync nft ${nft.identifier} in ${nft.chain.name} with type ${nft.type::class.simpleName}", it)
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package io.novafoundation.nova.feature_nft_impl.data.source

import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.async
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import java.util.Collections
import java.util.concurrent.ConcurrentHashMap
import kotlin.coroutines.coroutineContext

class JobOrchestrator {

private val runningJobs: MutableSet<String> = Collections.newSetFromMap(ConcurrentHashMap())

private val mutex = Mutex()

suspend fun runUniqueJob(id: String, action: suspend () -> Unit) = mutex.withLock {
if (id in runningJobs) {
return@withLock
}

runningJobs += id

CoroutineScope(coroutineContext).async { action() }
.invokeOnCompletion { runningJobs -= id }
}
}
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
package io.novafoundation.nova.feature_nft_impl.data.source

import io.novafoundation.nova.feature_account_api.domain.model.MetaAccount
import io.novafoundation.nova.feature_nft_api.data.model.Nft
import io.novafoundation.nova.runtime.multiNetwork.chain.model.Chain

interface NftProvider {

suspend fun initialNftsSync(chain: Chain, metaAccount: MetaAccount)

suspend fun nftFullSync(nft: Nft)
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.novafoundation.nova.feature_nft_impl.data.source

import io.novafoundation.nova.feature_nft_api.data.model.Nft
import io.novafoundation.nova.feature_nft_impl.data.source.providers.rmrkV1.RmrkV1NftProvider
import io.novafoundation.nova.feature_nft_impl.data.source.providers.rmrkV2.RmrkV2NftProvider
import io.novafoundation.nova.feature_nft_impl.data.source.providers.uniques.UniquesNftProvider
Expand All @@ -8,16 +9,27 @@ import io.novafoundation.nova.runtime.ext.genesisHash
import io.novafoundation.nova.runtime.multiNetwork.chain.model.Chain

class NftProvidersRegistry(
private val uniquesNftSource: UniquesNftProvider,
private val uniquesNftProvider: UniquesNftProvider,
private val rmrkV1NftProvider: RmrkV1NftProvider,
private val rmrkV2NftProvider: RmrkV2NftProvider,
) {

private val statemineProviders = listOf(uniquesNftProvider)
private val kusamaProviders = listOf(rmrkV1NftProvider, rmrkV2NftProvider)

fun get(chain: Chain): List<NftProvider> {
return when (chain.genesisHash) {
Chain.Geneses.STATEMINE -> listOf(uniquesNftSource)
Chain.Geneses.KUSAMA -> listOf(rmrkV1NftProvider, rmrkV2NftProvider)
Chain.Geneses.STATEMINE -> statemineProviders
Chain.Geneses.KUSAMA -> kusamaProviders
else -> emptyList()
}
}

fun get(nft: Nft): NftProvider {
return when(nft.type) {
is Nft.Type.Rmrk1 -> rmrkV1NftProvider
is Nft.Type.Rmrk2 -> rmrkV2NftProvider
is Nft.Type.Uniques -> uniquesNftProvider
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import io.novafoundation.nova.core_db.dao.NftDao
import io.novafoundation.nova.core_db.model.NftLocal
import io.novafoundation.nova.feature_account_api.domain.model.MetaAccount
import io.novafoundation.nova.feature_account_api.domain.model.addressIn
import io.novafoundation.nova.feature_nft_api.data.model.Nft
import io.novafoundation.nova.feature_nft_impl.data.source.NftProvider
import io.novafoundation.nova.feature_nft_impl.data.source.providers.rmrkV1.network.RmrkV1Api
import io.novafoundation.nova.runtime.multiNetwork.chain.model.Chain
Expand Down Expand Up @@ -43,6 +44,10 @@ class RmrkV1NftProvider(
nftDao.insertNftsDiff(NftLocal.Type.RMRK1, metaAccount.id, toSave)
}

override suspend fun nftFullSync(nft: Nft) {
// TODO
}

private fun identifier(chainId: ChainId, id: String): String {
return "$chainId-$id"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import io.novafoundation.nova.core_db.dao.NftDao
import io.novafoundation.nova.core_db.model.NftLocal
import io.novafoundation.nova.feature_account_api.domain.model.MetaAccount
import io.novafoundation.nova.feature_account_api.domain.model.addressIn
import io.novafoundation.nova.feature_nft_api.data.model.Nft
import io.novafoundation.nova.feature_nft_impl.data.source.NftProvider
import io.novafoundation.nova.feature_nft_impl.data.source.providers.rmrkV2.network.RmrkV2Api
import io.novafoundation.nova.runtime.multiNetwork.chain.model.Chain
Expand Down Expand Up @@ -40,6 +41,10 @@ class RmrkV2NftProvider(
nftDao.insertNftsDiff(NftLocal.Type.RMRK2, metaAccount.id, toSave)
}

override suspend fun nftFullSync(nft: Nft) {
// TODO
}

private fun identifier(chainId: ChainId, nftId: String): String {
return "$chainId-$nftId"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@ import io.novafoundation.nova.core_db.dao.NftDao
import io.novafoundation.nova.core_db.model.NftLocal
import io.novafoundation.nova.feature_account_api.domain.model.MetaAccount
import io.novafoundation.nova.feature_account_api.domain.model.accountIdIn
import io.novafoundation.nova.feature_nft_api.data.model.Nft
import io.novafoundation.nova.feature_nft_impl.data.network.distributed.FileStorageAdapter.adoptFileStorageLinkToHttps
import io.novafoundation.nova.feature_nft_impl.data.source.NftProvider
import io.novafoundation.nova.feature_nft_impl.data.source.providers.uniques.network.IpfsApi
import io.novafoundation.nova.runtime.multiNetwork.chain.model.Chain
import io.novafoundation.nova.runtime.multiNetwork.chain.model.ChainId
import io.novafoundation.nova.runtime.storage.source.StorageDataSource
Expand All @@ -20,6 +23,7 @@ import java.math.BigInteger
class UniquesNftProvider(
private val remoteStorage: StorageDataSource,
private val nftDao: NftDao,
private val ipfsApi: IpfsApi,
) : NftProvider {

override suspend fun initialNftsSync(chain: Chain, metaAccount: MetaAccount) {
Expand Down Expand Up @@ -91,6 +95,26 @@ class UniquesNftProvider(
nftDao.insertNftsDiff(NftLocal.Type.UNIQUES, metaAccount.id, newNfts)
}

override suspend fun nftFullSync(nft: Nft) {
if (nft.metadataRaw == null) {
nftDao.markFullSynced(nft.identifier)

return
}

val metadataLink = nft.metadataRaw!!.decodeToString().adoptFileStorageLinkToHttps()
val metadata = ipfsApi.getIpfsContent(metadataLink)

nftDao.updateNft(nft.identifier) { local ->
local.copy(
name = metadata.name!!,
media = metadata.image?.adoptFileStorageLinkToHttps(),
label = metadata.description,
wholeDetailsLoaded = true
)
}
}

private fun identifier(chainId: ChainId, collectionId: BigInteger, instanceId: BigInteger): String {
return "$chainId-$collectionId-$instanceId"
}
Expand Down
Loading

0 comments on commit 7faf275

Please sign in to comment.