Skip to content

Fix tag updates #70

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 15, 2025
Merged

Fix tag updates #70

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...duckvba.Server/ContentSynchronization/Pipeline/Abstract/SynchronizationPipelineFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public ISynchronizationPipeline<SyncContext, bool> Create<TParameters>(TParamete
return parameters switch
{
XmldocSyncRequestParameters => new SynchronizeXmlPipeline(parameters, _logger, _content, _github, _merge, _staging, _markdown, tokenSource, _inspections, _quickfixes, _annotations, _annotationParser, _quickFixParser, _inspectionParser, _tagServices),
TagSyncRequestParameters => new SynchronizeTagsPipeline(parameters, _logger, _content, _github, _merge, _staging, tokenSource),
TagSyncRequestParameters => new SynchronizeTagsPipeline(parameters, _logger, _content, _tagServices, _github, _merge, _staging, tokenSource),
_ => throw new NotSupportedException(),
};
}
Expand Down
2 changes: 1 addition & 1 deletion ...erduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncTags/LoadGitHubTagsBlock.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ public LoadGitHubTagsBlock(PipelineSection<SyncContext> parent, CancellationToke

public override async Task<SyncContext> TransformAsync(SyncContext input)
{
var githubTags = await _github.GetAllTagsAsync(Context.RubberduckDbMain.Name);
var githubTags = await _github.GetAllTagsAsync();
var (gitHubMain, gitHubNext, gitHubOthers) = githubTags.GetLatestTags();

Context.LoadGitHubTags(gitHubMain, gitHubNext, gitHubOthers);
Expand Down
185 changes: 124 additions & 61 deletions rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncTags/SyncTagsSection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,84 +2,147 @@
using rubberduckvba.Server.ContentSynchronization.Pipeline.Sections.Context;
using rubberduckvba.Server.Model;
using rubberduckvba.Server.Services;
using rubberduckvba.Server.Services.rubberduckdb;
using System.Threading.Tasks.Dataflow;

namespace rubberduckvba.Server.ContentSynchronization.Pipeline.Sections.SyncTags;

public class SyncTagsSection : PipelineSection<SyncContext>
{
public SyncTagsSection(IPipeline<SyncContext, bool> parent, CancellationTokenSource tokenSource, ILogger logger, IRubberduckDbService content, IGitHubClientService github, IStagingServices staging)
public SyncTagsSection(IPipeline<SyncContext, bool> parent, CancellationTokenSource tokenSource, ILogger logger, TagServices tagServices, IGitHubClientService github, IStagingServices staging)
: base(parent, tokenSource, logger)
{
ReceiveRequest = new ReceiveRequestBlock(this, tokenSource, logger);
BroadcastParameters = new BroadcastParametersBlock(this, tokenSource, logger);
AcquireDbMainTag = new AcquireDbMainTagGraphBlock(this, tokenSource, content, logger);
AcquireDbNextTag = new AcquireDbNextTagGraphBlock(this, tokenSource, content, logger);
JoinDbTags = new DataflowJoinBlock<TagGraph, TagGraph>(this, tokenSource, logger, nameof(JoinDbTags));
LoadDbTags = new LoadDbLatestTagsBlock(this, tokenSource, logger);
LoadGitHubTags = new LoadGitHubTagsBlock(this, tokenSource, github, logger);
JoinTags = new DataflowJoinBlock<SyncContext, SyncContext>(this, tokenSource, logger, nameof(JoinTags));
BroadcastTags = new BroadcastTagsBlock(this, tokenSource, logger);
StreamGitHubTags = new StreamGitHubTagsBlock(this, tokenSource, logger);
GetTagAssets = new GetTagAssetsBlock(this, tokenSource, github, logger);
TagBuffer = new TagBufferBlock(this, tokenSource, logger);
AccumulateProcessedTags = new AccumulateProcessedTagsBlock(this, tokenSource, logger);
SaveTags = new BulkSaveStagingBlock(this, tokenSource, staging, logger);
SynchronizeTags = new SynchronizeTagsBlock(this, tokenSource, logger, tagServices, github);
//ReceiveRequest = new ReceiveRequestBlock(this, tokenSource, logger);
//BroadcastParameters = new BroadcastParametersBlock(this, tokenSource, logger);
//AcquireDbMainTag = new AcquireDbMainTagGraphBlock(this, tokenSource, content, logger);
//AcquireDbNextTag = new AcquireDbNextTagGraphBlock(this, tokenSource, content, logger);
//JoinDbTags = new DataflowJoinBlock<TagGraph, TagGraph>(this, tokenSource, logger, nameof(JoinDbTags));
//LoadDbTags = new LoadDbLatestTagsBlock(this, tokenSource, logger);
//LoadGitHubTags = new LoadGitHubTagsBlock(this, tokenSource, github, logger);
//JoinTags = new DataflowJoinBlock<SyncContext, SyncContext>(this, tokenSource, logger, nameof(JoinTags));
//BroadcastTags = new BroadcastTagsBlock(this, tokenSource, logger);
//StreamGitHubTags = new StreamGitHubTagsBlock(this, tokenSource, logger);
//GetTagAssets = new GetTagAssetsBlock(this, tokenSource, github, logger);
//TagBuffer = new TagBufferBlock(this, tokenSource, logger);
//AccumulateProcessedTags = new AccumulateProcessedTagsBlock(this, tokenSource, logger);
//SaveTags = new BulkSaveStagingBlock(this, tokenSource, staging, logger);
}

#region blocks
private ReceiveRequestBlock ReceiveRequest { get; }
private BroadcastParametersBlock BroadcastParameters { get; }
private AcquireDbMainTagGraphBlock AcquireDbMainTag { get; }
private AcquireDbNextTagGraphBlock AcquireDbNextTag { get; }
private DataflowJoinBlock<TagGraph, TagGraph> JoinDbTags { get; }
private LoadDbLatestTagsBlock LoadDbTags { get; }
private LoadGitHubTagsBlock LoadGitHubTags { get; }
private DataflowJoinBlock<SyncContext, SyncContext> JoinTags { get; }
private BroadcastTagsBlock BroadcastTags { get; }
private StreamGitHubTagsBlock StreamGitHubTags { get; }
private GetTagAssetsBlock GetTagAssets { get; }
private TagBufferBlock TagBuffer { get; }
private AccumulateProcessedTagsBlock AccumulateProcessedTags { get; }
private BulkSaveStagingBlock SaveTags { get; }
//#region blocks
private SynchronizeTagsBlock SynchronizeTags { get; }
//private ReceiveRequestBlock ReceiveRequest { get; }
//private BroadcastParametersBlock BroadcastParameters { get; }
//private AcquireDbMainTagGraphBlock AcquireDbMainTag { get; }
//private AcquireDbNextTagGraphBlock AcquireDbNextTag { get; }
//private DataflowJoinBlock<TagGraph, TagGraph> JoinDbTags { get; }
//private LoadDbLatestTagsBlock LoadDbTags { get; }
//private LoadGitHubTagsBlock LoadGitHubTags { get; }
//private DataflowJoinBlock<SyncContext, SyncContext> JoinTags { get; }
//private BroadcastTagsBlock BroadcastTags { get; }
//private StreamGitHubTagsBlock StreamGitHubTags { get; }
//private GetTagAssetsBlock GetTagAssets { get; }
//private TagBufferBlock TagBuffer { get; }
//private AccumulateProcessedTagsBlock AccumulateProcessedTags { get; }
//private BulkSaveStagingBlock SaveTags { get; }

public ITargetBlock<SyncRequestParameters> InputBlock => ReceiveRequest.Block;
public Task OutputTask => SaveTags.Block.Completion;
public ITargetBlock<TagSyncRequestParameters> InputBlock => SynchronizeTags.Block!;
public Task OutputTask => SynchronizeTags.Block.Completion;

protected override IReadOnlyDictionary<string, IDataflowBlock> Blocks => new Dictionary<string, IDataflowBlock>
{
[nameof(ReceiveRequest)] = ReceiveRequest.Block,
[nameof(BroadcastParameters)] = BroadcastParameters.Block,
[nameof(AcquireDbMainTag)] = AcquireDbMainTag.Block,
[nameof(AcquireDbNextTag)] = AcquireDbNextTag.Block,
[nameof(JoinDbTags)] = JoinDbTags.Block,
[nameof(LoadDbTags)] = LoadDbTags.Block,
[nameof(LoadGitHubTags)] = LoadGitHubTags.Block,
[nameof(JoinTags)] = JoinTags.Block,
[nameof(BroadcastTags)] = BroadcastTags.Block,
[nameof(StreamGitHubTags)] = StreamGitHubTags.Block,
[nameof(GetTagAssets)] = GetTagAssets.Block,
[nameof(TagBuffer)] = TagBuffer.Block,
[nameof(AccumulateProcessedTags)] = AccumulateProcessedTags.Block,
[nameof(SaveTags)] = SaveTags.Block,
[nameof(SynchronizeTags)] = SynchronizeTags.Block,
// [nameof(ReceiveRequest)] = ReceiveRequest.Block,
// [nameof(BroadcastParameters)] = BroadcastParameters.Block,
// [nameof(AcquireDbMainTag)] = AcquireDbMainTag.Block,
// [nameof(AcquireDbNextTag)] = AcquireDbNextTag.Block,
// [nameof(JoinDbTags)] = JoinDbTags.Block,
// [nameof(LoadDbTags)] = LoadDbTags.Block,
// [nameof(LoadGitHubTags)] = LoadGitHubTags.Block,
// [nameof(JoinTags)] = JoinTags.Block,
// [nameof(BroadcastTags)] = BroadcastTags.Block,
// [nameof(StreamGitHubTags)] = StreamGitHubTags.Block,
// [nameof(GetTagAssets)] = GetTagAssets.Block,
// [nameof(TagBuffer)] = TagBuffer.Block,
// [nameof(AccumulateProcessedTags)] = AccumulateProcessedTags.Block,
// [nameof(SaveTags)] = SaveTags.Block,
};
#endregion
//#endregion

public override void CreateBlocks()
{
ReceiveRequest.CreateBlock();
BroadcastParameters.CreateBlock(ReceiveRequest);
AcquireDbMainTag.CreateBlock(BroadcastParameters);
AcquireDbNextTag.CreateBlock(BroadcastParameters);
JoinDbTags.CreateBlock(AcquireDbMainTag, AcquireDbNextTag);
LoadDbTags.CreateBlock(JoinDbTags);
LoadGitHubTags.CreateBlock(LoadDbTags);
JoinTags.CreateBlock(LoadDbTags, LoadGitHubTags);
BroadcastTags.CreateBlock(JoinTags);
StreamGitHubTags.CreateBlock(BroadcastTags);
GetTagAssets.CreateBlock(StreamGitHubTags);
TagBuffer.CreateBlock(GetTagAssets);
AccumulateProcessedTags.CreateBlock(TagBuffer);
SaveTags.CreateBlock(AccumulateProcessedTags);
SynchronizeTags.CreateBlock();
//ReceiveRequest.CreateBlock();
//BroadcastParameters.CreateBlock(ReceiveRequest);
//AcquireDbMainTag.CreateBlock(BroadcastParameters);
//AcquireDbNextTag.CreateBlock(BroadcastParameters);
//JoinDbTags.CreateBlock(AcquireDbMainTag, AcquireDbNextTag);
//LoadDbTags.CreateBlock(JoinDbTags);
//LoadGitHubTags.CreateBlock(LoadDbTags);
//JoinTags.CreateBlock(LoadDbTags, LoadGitHubTags);
//BroadcastTags.CreateBlock(JoinTags);
//StreamGitHubTags.CreateBlock(BroadcastTags);
//GetTagAssets.CreateBlock(StreamGitHubTags);
//TagBuffer.CreateBlock(GetTagAssets);
//AccumulateProcessedTags.CreateBlock(TagBuffer);
//SaveTags.CreateBlock(AccumulateProcessedTags);
}
}

public class SynchronizeTagsBlock : ActionBlockBase<TagSyncRequestParameters, SyncContext>
{
private readonly IGitHubClientService _github;
private readonly TagServices _tagServices;

public SynchronizeTagsBlock(PipelineSection<SyncContext> parent, CancellationTokenSource tokenSource, ILogger logger,
TagServices tagServices,
IGitHubClientService github)
: base(parent, tokenSource, logger)
{
_tagServices = tagServices;
_github = github;
}

protected override async Task ActionAsync(TagSyncRequestParameters input)
{
var getGithubTags = _github.GetAllTagsAsync();
var dbMain = _tagServices.GetLatestTag(false);
var dbNext = _tagServices.GetLatestTag(true);

var githubTags = await getGithubTags;
var (gitHubMain, gitHubNext, _) = githubTags.GetLatestTags();

var mergedMain = (dbMain ?? gitHubMain with { InstallerDownloads = gitHubMain.InstallerDownloads })!;
var mergedNext = (dbNext ?? gitHubNext with { InstallerDownloads = gitHubNext.InstallerDownloads })!;

var inserts = new List<TagGraph>();
var updates = new List<TagGraph>();

if (dbMain is null)
{
inserts.Add(mergedMain);
}
else
{
updates.Add(mergedMain);
}

if (dbNext is null)
{
inserts.Add(mergedNext);
}
else
{
updates.Add(mergedNext);
}

if (inserts.Any())
{
_tagServices.Create(inserts);
}
if (updates.Any())
{
_tagServices.Update(updates);
}
}
}
2 changes: 1 addition & 1 deletion ...erduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/SyncXmldocSection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ protected override async Task ActionAsync(SyncRequestParameters input)
var dbMain = await _content.GetLatestTagAsync(RepositoryId.Rubberduck, includePreRelease: false);
Context.LoadRubberduckDbMain(dbMain);

var githubTags = await _github.GetAllTagsAsync(dbMain.Name);
var githubTags = await _github.GetAllTagsAsync();
// LoadInspectionDefaultConfig
var config = await _github.GetCodeAnalysisDefaultsConfigAsync();
Context.LoadInspectionDefaultConfig(config);
Expand Down
10 changes: 7 additions & 3 deletions rubberduckvba.Server/ContentSynchronization/Pipeline/SynchronizeTagsPipeline.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using rubberduckvba.Server.ContentSynchronization.Pipeline.Sections.SyncTags;
using rubberduckvba.Server.ContentSynchronization.XmlDoc.Abstract;
using rubberduckvba.Server.Services;
using rubberduckvba.Server.Services.rubberduckdb;

namespace rubberduckvba.Server.ContentSynchronization.Pipeline;

Expand All @@ -12,14 +13,17 @@ public class SynchronizeTagsPipeline : PipelineBase<SyncContext, bool>, ISynchro
private readonly IGitHubClientService _github;
private readonly IXmlDocMerge _merge;
private readonly IStagingServices _staging;
private readonly TagServices _tagServices;

public SynchronizeTagsPipeline(IRequestParameters parameters, ILogger logger, IRubberduckDbService content, IGitHubClientService github, IXmlDocMerge merge, IStagingServices staging, CancellationTokenSource tokenSource)
public SynchronizeTagsPipeline(IRequestParameters parameters, ILogger logger,
IRubberduckDbService content, TagServices tagServices, IGitHubClientService github, IXmlDocMerge merge, IStagingServices staging, CancellationTokenSource tokenSource)
: base(new SyncContext(parameters), tokenSource, logger)
{
_content = content;
_github = github;
_merge = merge;
_staging = staging;
_tagServices = tagServices;
}

public async Task<IPipelineResult<bool>> ExecuteAsync(SyncRequestParameters parameters, CancellationTokenSource tokenSource)
Expand All @@ -31,14 +35,14 @@ public async Task<IPipelineResult<bool>> ExecuteAsync(SyncRequestParameters para
}

// 01. Create the pipeline sections
var synchronizeTags = new SyncTagsSection(this, tokenSource, Logger, _content, _github, _staging);
var synchronizeTags = new SyncTagsSection(this, tokenSource, Logger, _tagServices, _github, _staging);

// 02. Wire up the pipeline
AddSections(parameters, synchronizeTags);
DisposeAfter(synchronizeTags.WhenAllBlocksCompleted);

// 03. Light it up
Start(synchronizeTags.InputBlock, parameters);
Start(synchronizeTags.InputBlock, (TagSyncRequestParameters)parameters);

// 04. await completion
await synchronizeTags.OutputTask;
Expand Down
10 changes: 5 additions & 5 deletions rubberduckvba.Server/Services/GitHubClientService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ namespace rubberduckvba.Server.Services;
public interface IGitHubClientService
{
Task<ClaimsPrincipal?> ValidateTokenAsync(string token);
Task<IEnumerable<TagGraph>> GetAllTagsAsync(string? dbMainTagName);
Task<IEnumerable<TagGraph>> GetAllTagsAsync();
Task<TagGraph> GetTagAsync(string? token, string name);
Task<IEnumerable<InspectionDefaultConfig>> GetCodeAnalysisDefaultsConfigAsync();
}
Expand Down Expand Up @@ -60,18 +60,18 @@ private class ReleaseComparer : IEqualityComparer<Release>
return new ClaimsPrincipal(identity);
}

public async Task<IEnumerable<TagGraph>> GetAllTagsAsync(string? dbMainTagName)
public async Task<IEnumerable<TagGraph>> GetAllTagsAsync()
{
var config = configuration.Value;
var credentials = new Credentials(config.OrgToken);
var client = new GitHubClient(new ProductHeaderValue(config.UserAgent), new InMemoryCredentialStore(credentials));


var getReleases = client.Repository.Release.GetAll(config.OwnerOrg, config.Rubberduck, new ApiOptions { PageCount = 1, PageSize = 10 });
var getKnownMain = client.Repository.Release.Get(config.OwnerOrg, config.Rubberduck, dbMainTagName);
await Task.WhenAll(getReleases, getKnownMain);
var getLatest = client.Repository.Release.GetLatest(config.OwnerOrg, config.Rubberduck);
await Task.WhenAll(getReleases, getLatest);

var releases = (await getReleases).Append(await getKnownMain).ToHashSet(new ReleaseComparer());
var releases = (await getReleases).Append(await getLatest).ToHashSet(new ReleaseComparer());

return (from release in releases
let installer = release.Assets.SingleOrDefault(asset => asset.Name.EndsWith(".exe") && asset.Name.StartsWith("Rubberduck.Setup"))
Expand Down
6 changes: 5 additions & 1 deletion rubberduckvba.Server/Services/rubberduckdb/TagServices.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,13 @@ public IEnumerable<Tag> GetLatestTags(IEnumerable<Tag> allTags) => allTags
.SelectMany(tags => tags.Take(1))
.ToList();

public TagGraph GetLatestTag(bool isPreRelease)
public TagGraph? GetLatestTag(bool isPreRelease)
{
var latestTags = GetLatestTags();
if (!latestTags.Any())
{
return null;
}

if (!isPreRelease)
{
Expand Down