From 74e2e0bed8be609173a72f3e1306199ba633c6ec Mon Sep 17 00:00:00 2001 From: Thomas Hounsell Date: Wed, 17 May 2017 11:48:39 +0100 Subject: [PATCH] Rework front page to scale much better Old page took anywhere up to 20s to generate with a 100k builds collection. This cuts it down to between 1s-2s. --- BuildFeed.Model/BuildRepository.cs | 124 +++++++++++++---------- BuildFeed.Model/View/FrontPage.cs | 47 ++++++++- BuildFeed/Controllers/frontController.cs | 8 +- 3 files changed, 113 insertions(+), 66 deletions(-) diff --git a/BuildFeed.Model/BuildRepository.cs b/BuildFeed.Model/BuildRepository.cs index 0c4cf5c..c155c6a 100644 --- a/BuildFeed.Model/BuildRepository.cs +++ b/BuildFeed.Model/BuildRepository.cs @@ -153,74 +153,90 @@ public async Task> SelectBuildsByOrder(int limit = -1, int skip = 0) } [DataObjectMethod(DataObjectMethodType.Select, false)] - public async Task SelectFrontPage(ProjectFamily family) + public async Task> SelectFrontPage() { - FrontPage fp = new FrontPage(); + var families = new Dictionary(); - IFindFluent query = _buildCollection.Find(new BsonDocument - { - {"$where", "!this.LabUrl.contains(\"xbox\")"}, + IAggregateFluent query = _buildCollection.Aggregate() + .Match(new BsonDocument { - nameof(Build.Family), family - } - }).Sort(sortByCompileDate).Limit(1); - fp.CurrentCanary = await query.FirstOrDefaultAsync(); - - query = _buildCollection.Find(new BsonDocument - { - {"$where", "!this.LabUrl.contains(\"xbox\")"}, - { - nameof(Build.Family), family - }, - { - nameof(Build.MajorVersion), 10 - }, - { - nameof(Build.SourceType), new BsonDocument { + nameof(Build.Family), new BsonDocument + { + {"$gte", 30} + } + } + }) + .Group(new BsonDocument + { + { + "_id", new BsonDocument + { + {nameof(Build.Family), $"${nameof(Build.Family)}"}, + {nameof(Build.LabUrl), $"${nameof(Build.LabUrl)}"}, + {nameof(Build.SourceType), $"${nameof(Build.SourceType)}"} + } + }, + { + "items", new BsonDocument { - "$in", new BsonArray { - TypeOfSource.PublicRelease, - TypeOfSource.UpdateGDR + "$push", new BsonDocument + { + {nameof(Build.Id), "$_id"}, + {nameof(Build.MajorVersion), $"${nameof(Build.MajorVersion)}"}, + {nameof(Build.MinorVersion), $"${nameof(Build.MinorVersion)}"}, + {nameof(Build.Number), $"${nameof(Build.Number)}"}, + {nameof(Build.Revision), $"${nameof(Build.Revision)}"}, + {nameof(Build.Lab), $"${nameof(Build.Lab)}"}, + {nameof(Build.BuildTime), $"${nameof(Build.BuildTime)}"} + } } } } - } - }).Sort(sortByCompileDate).Limit(1); - fp.CurrentInsider = await query.FirstOrDefaultAsync(); + }); - query = _buildCollection.Find(new BsonDocument + var dbResults = await query.ToListAsync(); + + var results = from g in dbResults + select new + { + Key = new + { + Family = (ProjectFamily)g["_id"].AsBsonDocument[nameof(Build.Family)].AsInt32, + LabUrl = g["_id"].AsBsonDocument[nameof(Build.LabUrl)].AsString, + SourceType = (TypeOfSource)g["_id"].AsBsonDocument[nameof(Build.SourceType)].AsInt32 + }, + + Items = from i in g["items"].AsBsonArray + select new FrontPageBuild + { + Id = i[nameof(Build.Id)].AsGuid, + MajorVersion = (uint)i[nameof(Build.MajorVersion)].AsInt32, + MinorVersion = (uint)i[nameof(Build.MinorVersion)].AsInt32, + Number = (uint)i[nameof(Build.Number)].AsInt32, + Revision = (uint?)i[nameof(Build.Revision)].AsNullableInt32, + Lab = i[nameof(Build.Lab)].AsString, + BuildTime = i[nameof(Build.BuildTime)].ToNullableUniversalTime() + } + }; + + IEnumerable listOfFamilies = results.GroupBy(g => g.Key.Family).Select(g => g.Key); + + foreach (ProjectFamily family in listOfFamilies) { - {"$where", "((this.MajorVersion === 10 && this.LabUrl.contains(\"_release\")) || this.MajorVersion < 10) && !this.LabUrl.contains(\"xbox\")"}, + FrontPage fp = new FrontPage { - nameof(Build.Family), family - }, - { - nameof(Build.SourceType), new BsonDocument - { - { - "$in", new BsonArray - { - TypeOfSource.PublicRelease, - TypeOfSource.UpdateGDR - } - } - } - } - }).Sort(sortByCompileDate).Limit(1); - fp.CurrentRelease = await query.FirstOrDefaultAsync(); + CurrentCanary = results.Where(g => g.Key.Family == family && !g.Key.LabUrl.Contains("xbox")).SelectMany(g => g.Items).OrderByDescending(b => b.BuildTime).FirstOrDefault(), + CurrentInsider = results.Where(g => g.Key.Family == family && !g.Key.LabUrl.Contains("xbox") && (g.Key.SourceType == TypeOfSource.PublicRelease || g.Key.SourceType == TypeOfSource.UpdateGDR)).SelectMany(g => g.Items).OrderByDescending(b => b.BuildTime).FirstOrDefault(), + CurrentRelease = results.Where(g => g.Key.Family == family && g.Key.LabUrl.Contains("_release") && !g.Key.LabUrl.Contains("xbox") && (g.Key.SourceType == TypeOfSource.PublicRelease || g.Key.SourceType == TypeOfSource.UpdateGDR)).SelectMany(g => g.Items).OrderByDescending(b => b.BuildTime).FirstOrDefault(), + CurrentXbox = results.Where(g => g.Key.Family == family && g.Key.LabUrl.Contains("xbox")).SelectMany(g => g.Items).OrderByDescending(b => b.BuildTime).FirstOrDefault() + }; - query = _buildCollection.Find(new BsonDocument - { - {"$where", "this.LabUrl.contains(\"xbox\")"}, - { - nameof(Build.Family), family - } - }).Sort(sortByCompileDate).Limit(1); - fp.CurrentXbox = await query.FirstOrDefaultAsync(); + families.Add(family, fp); + } - return fp; + return families; } [DataObjectMethod(DataObjectMethodType.Select, false)] diff --git a/BuildFeed.Model/View/FrontPage.cs b/BuildFeed.Model/View/FrontPage.cs index c6f10bb..06b63c0 100644 --- a/BuildFeed.Model/View/FrontPage.cs +++ b/BuildFeed.Model/View/FrontPage.cs @@ -1,10 +1,47 @@ -namespace BuildFeed.Model.View +using System; +using System.ComponentModel.DataAnnotations; +using BuildFeed.Local; +using MongoDB.Bson.Serialization.Attributes; + +namespace BuildFeed.Model.View { public class FrontPage { - public Build CurrentCanary { get; set; } - public Build CurrentInsider { get; set; } - public Build CurrentRelease { get; set; } - public Build CurrentXbox { get; set; } + public FrontPageBuild CurrentCanary { get; set; } + public FrontPageBuild CurrentInsider { get; set; } + public FrontPageBuild CurrentRelease { get; set; } + public FrontPageBuild CurrentXbox { get; set; } + } + + public class FrontPageBuild + { + [Key] + [BsonId] + public Guid Id { get; set; } + + [Required] + [Display(ResourceType = typeof(VariantTerms), Name = nameof(VariantTerms.Model_MajorVersion))] + public uint MajorVersion { get; set; } + + [Required] + [Display(ResourceType = typeof(VariantTerms), Name = nameof(VariantTerms.Model_MinorVersion))] + public uint MinorVersion { get; set; } + + [Required] + [Display(ResourceType = typeof(VariantTerms), Name = nameof(VariantTerms.Model_BuildNumber))] + public uint Number { get; set; } + + [Display(ResourceType = typeof(VariantTerms), Name = nameof(VariantTerms.Model_Revision))] + [DisplayFormat(ConvertEmptyStringToNull = true)] + public uint? Revision { get; set; } + + [Display(ResourceType = typeof(VariantTerms), Name = nameof(VariantTerms.Model_LabString))] + public string Lab { get; set; } + + [Display(ResourceType = typeof(VariantTerms), Name = nameof(VariantTerms.Model_BuildTime))] + [DisplayFormat(ConvertEmptyStringToNull = true, + ApplyFormatInEditMode = true, + DataFormatString = "{0:yyMMdd-HHmm}")] + public DateTime? BuildTime { get; set; } } } \ No newline at end of file diff --git a/BuildFeed/Controllers/frontController.cs b/BuildFeed/Controllers/frontController.cs index 9ecf09e..6043fb5 100644 --- a/BuildFeed/Controllers/frontController.cs +++ b/BuildFeed/Controllers/frontController.cs @@ -38,13 +38,7 @@ public FrontController() #endif public async Task Index() { - var items = new Dictionary - { - {ProjectFamily.Redstone3, await _bModel.SelectFrontPage(ProjectFamily.Redstone3)}, - {ProjectFamily.Feature2, await _bModel.SelectFrontPage(ProjectFamily.Feature2)}, - {ProjectFamily.Redstone2, await _bModel.SelectFrontPage(ProjectFamily.Redstone2)}, - {ProjectFamily.Redstone, await _bModel.SelectFrontPage(ProjectFamily.Redstone)} - }; + Dictionary items = await _bModel.SelectFrontPage(); return View(nameof(Index), items); }