StarMirror/BuildFeed
1
0
Fork 0
mirror of https://gitlab.com/buildfeed/BuildFeed.git synced 2024-03-22 21:10:34 +08:00
Code Issues Projects Releases Wiki Activity

A fuckload of Mongo

Browse Source 
Starting to take shape with some nice perf gains.
This commit is contained in:
Thomas Hounsell 2015-09-09 23:48:29 +01:00
parent 34141f60f1
commit 2919fbea16
17 changed files with 850 additions and 645 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
BuildFeed/Controllers/frontController.cs
View File

@ -27,35 +27,20 @@ public class frontController : Controller
#endif
public ActionResult indexPage(int page)
{
var buildGroups = (from b in new Build().Select()
group b by new BuildGroup
{
Major = b.MajorVersion,
Minor = b.MinorVersion,
Build = b.Number,
Revision = b.Revision
}
into bg
orderby bg.Key.Major descending,
bg.Key.Minor descending,
bg.Key.Build descending,
bg.Key.Revision descending
select new FrontBuildGroup
{
Key = bg.Key,
LastBuild = bg.Max(m => m.BuildTime),
BuildCount = bg.Count()
}).ToArray();
Build bModel = new Build();
var buildGroups = bModel.SelectBuildGroups(PAGE_SIZE, (page - 1) * PAGE_SIZE);
ViewBag.PageNumber = page;
ViewBag.PageCount = Math.Ceiling(Convert.ToDouble(buildGroups.Length) / Convert.ToDouble(PAGE_SIZE));
ViewBag.PageCount = Math.Ceiling(
Convert.ToDouble(bModel.SelectBuildGroupsCount()) /
Convert.ToDouble(PAGE_SIZE));
if (ViewBag.PageNumber > ViewBag.PageCount)
if (ViewBag.PageCount != 0 && ViewBag.PageNumber > ViewBag.PageCount)
{
return new HttpNotFoundResult();
}
return View("index", buildGroups.Skip((page - 1) * PAGE_SIZE).Take(PAGE_SIZE));
return View("index", buildGroups);
}
[Route("group/{major}.{minor}.{number}.{revision}/")]
@ -64,23 +49,17 @@ bg.Key.Revision descending
#endif
public ActionResult viewGroup(byte major, byte minor, ushort number, ushort? revision = null)
{
var builds = (from b in new Build().Select()
group b by new BuildGroup
Build bModel = new Build();
var builds = bModel.SelectSingleBuildGroup(new BuildGroup()
{
Major = b.MajorVersion,
Minor = b.MinorVersion,
Build = b.Number,
Revision = b.Revision
}
into bg
where bg.Key.Major == major
where bg.Key.Minor == minor
where bg.Key.Build == number
where bg.Key.Revision == revision
select bg).Single();
Major = major,
Minor = minor,
Build = number,
Revision = revision
});
return builds.Count() == 1 ?
RedirectToAction("viewBuild", new {id = builds.Single().Id}) as ActionResult :
return builds.Item2.Count() == 1 ?
RedirectToAction("viewBuild", new { id = builds.Item2.Single().Id }) as ActionResult :
View(builds);
}
@ -149,17 +128,17 @@ public ActionResult viewLabPage(string lab, int page)
});
ViewBag.ItemId = lab;
var builds = new Build().SelectInBuildOrder().Where(b => b.Lab != null && (b.Lab.ToLower() == lab.ToLower())).ToArray();
var builds = new Build().SelectLabInBuildOrder(lab, (page - 1) * PAGE_SIZE, PAGE_SIZE);
ViewBag.PageNumber = page;
ViewBag.PageCount = Math.Ceiling(Convert.ToDouble(builds.Length) / Convert.ToDouble(PAGE_SIZE));
ViewBag.PageCount = Math.Ceiling(Convert.ToDouble(builds.Count) / Convert.ToDouble(PAGE_SIZE));
if (ViewBag.PageNumber > ViewBag.PageCount)
{
return new HttpNotFoundResult();
}
return View("viewLab", builds.Skip((page - 1) * PAGE_SIZE).Take(PAGE_SIZE));
return View("viewLab", builds);
}
[Route("source/{source}/", Order = 1, Name = "Source Root")]

113
BuildFeed/Models/Build.cs
View File

@ -1,16 +1,16 @@
using System;
using BuildFeed.Local;
using BuildFeed.Models.ViewModel.Front;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.ComponentModel.DataAnnotations;
using System.Linq;
using System.Text;
using Required = System.ComponentModel.DataAnnotations.RequiredAttribute;
using System.Web.Mvc;
using BuildFeed.Local;
using MongoDB.Driver;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using Required = System.ComponentModel.DataAnnotations.RequiredAttribute;
namespace BuildFeed.Models
{
@ -133,13 +133,78 @@ public Build()
}
[DataObjectMethod(DataObjectMethodType.Select, true)]
public IEnumerable<BuildModel> Select()
public List<BuildModel> Select()
{
var task = _buildCollection.Find(new BsonDocument()).ToListAsync();
task.Wait();
return task.Result;
}
[DataObjectMethod(DataObjectMethodType.Select, false)]
public List<FrontBuildGroup> SelectBuildGroups(int limit, int skip)
{
var pipeline = _buildCollection.Aggregate()
.Group(b => new BuildGroup()
{
Major = b.MajorVersion,
Minor = b.MinorVersion,
Build = b.Number,
Revision = b.Revision
},
bg => new FrontBuildGroup()
{
Key = bg.Key,
BuildCount = bg.Count(),
LastBuild = bg.Max(b => b.BuildTime)
})
.SortByDescending(b => b.Key.Major)
.ThenByDescending(b => b.Key.Minor)
.ThenByDescending(b => b.Key.Build)
.ThenByDescending(b => b.Key.Revision)
.Skip(skip)
.Limit(limit);
var task = pipeline.ToListAsync();
task.Wait();
return task.Result;
}
[DataObjectMethod(DataObjectMethodType.Select, true)]
public int SelectBuildGroupsCount()
{
var pipeline = _buildCollection.Aggregate()
.Group(b => new BuildGroup()
{
Major = b.MajorVersion,
Minor = b.MinorVersion,
Build = b.Number,
Revision = b.Revision
},
bg => new BsonDocument());
var task = pipeline.ToListAsync();
task.Wait();
return task.Result.Count();
}
[DataObjectMethod(DataObjectMethodType.Select, false)]
public Tuple<BuildGroup, IEnumerable<BuildModel>> SelectSingleBuildGroup(BuildGroup bGroup)
{
var pipeline = _buildCollection.Aggregate()
.Match(b => b.MajorVersion == bGroup.Major)
.Match(b => b.MinorVersion == bGroup.Minor)
.Match(b => b.Number == bGroup.Build)
.Match(b => b.Revision == bGroup.Revision)
.SortByDescending(b => b.BuildTime);
var task = pipeline.ToListAsync();
task.Wait();
return new Tuple<BuildGroup, IEnumerable<BuildModel>>(bGroup, task.Result);
}
[DataObjectMethod(DataObjectMethodType.Select, false)]
public BuildModel SelectById(Guid id)
{
@ -170,6 +235,22 @@ public IEnumerable<BuildModel> SelectInBuildOrder()
return task.Result;
}
[DataObjectMethod(DataObjectMethodType.Select, false)]
public List<BuildModel> SelectLabInBuildOrder(string lab, int skip, int limit)
{
var task = _buildCollection.Find(b => b.Lab != null && (b.Lab.ToLower() == lab.ToLower()))
.SortByDescending(b => b.BuildTime)
.ThenByDescending(b => b.MajorVersion)
.ThenByDescending(b => b.MinorVersion)
.ThenByDescending(b => b.Number)
.ThenByDescending(b => b.Revision)
.Skip(skip)
.Limit(limit)
.ToListAsync();
task.Wait();
return task.Result;
}
[DataObjectMethod(DataObjectMethodType.Select, false)]
public IEnumerable<BuildModel> SelectInVersionOrder()
{
@ -234,6 +315,13 @@ public void Insert(BuildModel item)
task.Wait();
}
[DataObjectMethod(DataObjectMethodType.Insert, false)]
public void InsertAll(IEnumerable<BuildModel> items)
{
var task = _buildCollection.InsertManyAsync(items);
task.Wait();
}
[DataObjectMethod(DataObjectMethodType.Update, true)]
public void Update(BuildModel item)
{
@ -245,13 +333,6 @@ public void Update(BuildModel item)
task.Wait();
}
[DataObjectMethod(DataObjectMethodType.Insert, false)]
public void InsertAll(IEnumerable<BuildModel> items)
{
var task = _buildCollection.InsertManyAsync(items);
task.Wait();
}
[DataObjectMethod(DataObjectMethodType.Delete, true)]
public void DeleteById(Guid id)
{
@ -316,7 +397,7 @@ public override string ToString()
}
}
public struct BuildGroup
public class BuildGroup
{
public byte Major { get; set; }
public byte Minor { get; set; }

2
BuildFeed/Views/front/editBuild.cshtml
View File

@ -1,4 +1,4 @@
@model BuildFeed.Models.Build
@model BuildFeed.Models.BuildModel
@{
ViewBag.Title = (string)(ViewContext.RouteData.Values["action"]) == "addBuild" ? string.Format("{0} | {1}", BuildFeed.Local.Front.AddABuild, BuildFeed.Local.Common.SiteName) : string.Format("{0} {1} | {2}", BuildFeed.Local.Front.EditBuild, Model.FullBuildString, BuildFeed.Local.Common.SiteName);
Html.EnableClientValidation();

2
BuildFeed/Views/front/viewBuild.cshtml
View File

@ -1,4 +1,4 @@
@model BuildFeed.Models.Build
@model BuildFeed.Models.BuildModel
@{
ViewBag.Title = Model.FullBuildString + " | BuildFeed";

8
BuildFeed/Views/front/viewGroup.cshtml
View File

@ -1,13 +1,13 @@
@model IGrouping<BuildFeed.Models.BuildGroup, BuildFeed.Models.Build>
@model Tuple<BuildFeed.Models.BuildGroup, IEnumerable<BuildFeed.Models.BuildModel>>
@using Humanizer;
@{
ViewBag.Title = string.Format("{0} | {1}", Model.Key.ToString(), BuildFeed.Local.Common.SiteName);
ViewBag.Title = string.Format("{0} | {1}", Model.Item1.ToString(), BuildFeed.Local.Common.SiteName);
}
<h1>@Model.Key.ToString()</h1>
<h1>@Model.Item1.ToString()</h1>
<div class="row">
@foreach (var build in Model.OrderBy(b => b.BuildTime))
@foreach (var build in Model.Item2)
{
<div class="col-sm-3 col-xs-6 build-group">
<h3 class="build-group-title no-wrapping" title="@build.Lab"><a href="@Url.Action("viewBuild", new {id = build.Id})">@build.Lab</a></h3>

2
BuildFeed/Views/front/viewLab.cshtml
View File

@ -1,4 +1,4 @@
@model IEnumerable<BuildFeed.Models.Build>
@model IEnumerable<BuildFeed.Models.BuildModel>
@using Humanizer;
@{
ViewBag.Title = string.Format("{0} {1}{2} | {3}", BuildFeed.Local.Front.BuildsFrom, ViewBag.ItemId, ViewBag.PageNumber == 1 ? "" : string.Format(BuildFeed.Local.Common.PageTitleSegment, ViewBag.PageNumber), BuildFeed.Local.Common.SiteName);

2
BuildFeed/Views/front/viewSource.cshtml
View File

@ -1,4 +1,4 @@
@model IEnumerable<BuildFeed.Models.Build>
@model IEnumerable<BuildFeed.Models.BuildModel>
@using Humanizer;
@{
ViewBag.Title = string.Format("{0}{1} | {2}", ViewBag.ItemId, ViewBag.PageNumber == 1 ? "" : string.Format(BuildFeed.Local.Common.PageTitleSegment, ViewBag.PageNumber), @BuildFeed.Local.Common.SiteName);

2
BuildFeed/Views/front/viewVersion.cshtml
View File

@ -1,4 +1,4 @@
@model IEnumerable<BuildFeed.Models.Build>
@model IEnumerable<BuildFeed.Models.BuildModel>
@using Humanizer;
@{
ViewBag.Title = string.Format("{0} {1}{2} | {3}", BuildFeed.Local.Common.ProductName, ViewBag.ItemId, ViewBag.PageNumber == 1 ? "" : string.Format(BuildFeed.Local.Common.PageTitleSegment, ViewBag.PageNumber), BuildFeed.Local.Common.SiteName);

2
BuildFeed/Views/front/viewYear.cshtml
View File

@ -1,4 +1,4 @@
@model IEnumerable<BuildFeed.Models.Build>
@model IEnumerable<BuildFeed.Models.BuildModel>
@using Humanizer;
@{
ViewBag.Title = string.Format("{0} {1}{2} | {3}", BuildFeed.Local.Front.BuildsFrom, ViewBag.ItemId, ViewBag.PageNumber == 1 ? "" : string.Format(BuildFeed.Local.Common.PageTitleSegment, ViewBag.PageNumber), BuildFeed.Local.Common.SiteName);

5
BuildFeed/Web.config
View File

@ -10,11 +10,8 @@
<add key="PreserveLoginUrl" value="true" />
<add key="ClientValidationEnabled" value="true" />
<add key="UnobtrusiveJavaScriptEnabled" value="true" />
<add key="data:ServerHost" value="localhost" />
<add key="data:ServerPort" value="6379" />
<add key="data:ServerDB" value="1" />
<add key="data:MongoDB" value="BuildFeed" />
<add key="RouteDebugger:Enabled" value="false" />
<add key="RouteDebugger:Enabled" value="true" />
</appSettings>
<system.web>
<compilation debug="true" targetFramework="4.6" />

10
BuildFeed/content/style.css
View File

@ -129,3 +129,13 @@ label, .control-label, .help-block, .checkbox, .radio
{
font-size: 16px;
}
#haackroutedebugger
{
color: #000;
}
#haackroutedebugger h1, #haackroutedebugger h2, #haackroutedebugger h3
{
text-shadow: none;
}

29
RedisMongoMigration/Build.cs Normal file
View File

@ -0,0 +1,29 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace RedisMongoMigration
{
public enum TypeOfSource
{
PublicRelease,
InternalLeak,
UpdateGDR,
UpdateLDR,
AppPackage,
BuildTools,
Documentation,
Logging,
PrivateLeak
}
public enum LevelOfFlight
{
None = 0,
Low = 1,
Medium = 2,
High = 3
}
}

110
RedisMongoMigration/Mongo/Build.cs Normal file
View File

@ -0,0 +1,110 @@
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using System;
using System.Collections.Generic;
using System.Text;
namespace RedisMongoMigration.Mongo
{
public class BuildModel
{
[BsonId]
public Guid Id { get; set; }
public long? LegacyId { get; set; }
public byte MajorVersion { get; set; }
public byte MinorVersion { get; set; }
public ushort Number { get; set; }
public ushort? Revision { get; set; }
public string Lab { get; set; }
public DateTime? BuildTime { get; set; }
public DateTime Added { get; set; }
public DateTime Modified { get; set; }
public TypeOfSource SourceType { get; set; }
public string SourceDetails { get; set; }
public DateTime? LeakDate { get; set; }
public LevelOfFlight FlightLevel { get; set; }
public bool IsLeaked
{
get
{
switch (SourceType)
{
case TypeOfSource.PublicRelease:
case TypeOfSource.InternalLeak:
case TypeOfSource.UpdateGDR:
case TypeOfSource.UpdateLDR:
return true;
default:
return false;
}
}
}
public string FullBuildString
{
get
{
StringBuilder sb = new StringBuilder();
sb.AppendFormat("{0}.{1}.{2}", MajorVersion, MinorVersion, Number);
if (Revision.HasValue)
{
sb.AppendFormat(".{0}", Revision);
}
if (!string.IsNullOrWhiteSpace(Lab))
{
sb.AppendFormat(".{0}", Lab);
}
if (BuildTime.HasValue)
{
sb.AppendFormat(".{0:yyMMdd-HHmm}", BuildTime);
}
return sb.ToString();
}
}
}
public class Build
{
private const string _buildCollectionName = "builds";
private MongoClient _dbClient;
private IMongoCollection<BuildModel> _buildCollection;
public Build()
{
_dbClient = new MongoClient(new MongoClientSettings()
{
Server = new MongoServerAddress("localhost", 27017)
});
_buildCollection = _dbClient.GetDatabase("BuildFeed").GetCollection<BuildModel>(_buildCollectionName);
}
public List<BuildModel> Select()
{
var task = _buildCollection.Find(new BsonDocument()).ToListAsync();
task.Wait();
return task.Result;
}
public void Insert(BuildModel item)
{
item.Id = Guid.NewGuid();
var task = _buildCollection.InsertOneAsync(item);
task.Wait();
}
public void InsertAll(IEnumerable<BuildModel> items)
{
var task = _buildCollection.InsertManyAsync(items);
task.Wait();
}
}
}

29
RedisMongoMigration/Program.cs
View File

@ -4,12 +4,41 @@
using System.Text;
using System.Threading.Tasks;
using MBuildModel = RedisMongoMigration.Mongo.BuildModel;
using MBuild = RedisMongoMigration.Mongo.Build;
using RBuild = RedisMongoMigration.Redis.Build;
namespace RedisMongoMigration
{
class Program
{
static void Main(string[] args)
{
var builds = RBuild.Select();
var newBuilds = from b in builds
select new MBuildModel()
{
Id = Guid.NewGuid(),
LegacyId = b.Id,
MajorVersion = b.MajorVersion,
MinorVersion = b.MinorVersion,
Number = b.Number,
Revision = b.Revision,
Lab = b.Lab,
BuildTime = b.BuildTime,
Added = b.Added,
Modified = b.Modified,
SourceType = b.SourceType,
SourceDetails = b.SourceDetails,
LeakDate = b.LeakDate,
FlightLevel = b.FlightLevel
};
MBuild m = new MBuild();
m.InsertAll(newBuilds);
Console.WriteLine("Builds: Complete");
Console.ReadKey();
}
}
}

62
RedisMongoMigration/Redis/Build.cs
View File

@ -1,6 +1,8 @@
using NServiceKit.DataAnnotations;
using NServiceKit.DesignPatterns.Model;
using NServiceKit.Redis;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.ComponentModel.DataAnnotations;
using System.Text;
@ -14,30 +16,18 @@ public class Build : IHasId<long>
[AutoIncrement]
[Index]
public long Id { get; set; }
public byte MajorVersion { get; set; }
public byte MinorVersion { get; set; }
public ushort Number { get; set; }
public ushort? Revision { get; set; }
public string Lab { get; set; }
public DateTime? BuildTime { get; set; }
public DateTime Added { get; set; }
public DateTime Modified { get; set; }
public TypeOfSource SourceType { get; set; }
public string SourceDetails { get; set; }
public DateTime? LeakDate { get; set; }
public LevelOfFlight FlightLevel { get; set; }
public bool IsLeaked
@ -81,52 +71,14 @@ public string FullBuildString
return sb.ToString();
}
}
}
public enum TypeOfSource
public static IEnumerable<Build> Select()
{
PublicRelease,
InternalLeak,
UpdateGDR,
UpdateLDR,
AppPackage,
BuildTools,
Documentation,
Logging,
PrivateLeak
}
public enum LevelOfFlight
using (RedisClient rClient = new RedisClient("localhost", 6379, db: 1))
{
None = 0,
Low = 1,
Medium = 2,
High = 3
}
public struct BuildVersion
{
public byte Major { get; set; }
public byte Minor { get; set; }
public override string ToString()
{
return $"{Major}.{Minor}";
}
}
public struct BuildGroup
{
public byte Major { get; set; }
public byte Minor { get; set; }
public ushort Build { get; set; }
public ushort? Revision { get; set; }
public override string ToString()
{
return Revision.HasValue ?
$"{Major}.{Minor}.{Build}.{Revision.Value}" :
$"{Major}.{Minor}.{Build}";
var client = rClient.As<Build>();
return client.GetAll();
}
}
}
}

15
RedisMongoMigration/RedisMongoMigration.csproj
View File

@ -33,6 +33,18 @@
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="MongoDB.Bson, Version=2.0.1.27, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\MongoDB.Bson.2.0.1\lib\net45\MongoDB.Bson.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="MongoDB.Driver, Version=2.0.1.27, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\MongoDB.Driver.2.0.1\lib\net45\MongoDB.Driver.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="MongoDB.Driver.Core, Version=2.0.1.27, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\MongoDB.Driver.Core.2.0.1\lib\net45\MongoDB.Driver.Core.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="NServiceKit.Common, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\NServiceKit.Common.1.0.31\lib\net35\NServiceKit.Common.dll</HintPath>
<Private>True</Private>
@ -60,6 +72,8 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Build.cs" />
<Compile Include="Mongo\Build.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Redis\Build.cs" />
@ -71,6 +85,7 @@
<None Include="App.config" />
<None Include="packages.config" />
</ItemGroup>
<ItemGroup />
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.

3
RedisMongoMigration/packages.config
View File

@ -1,5 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="MongoDB.Bson" version="2.0.1" targetFramework="net46" />
<package id="MongoDB.Driver" version="2.0.1" targetFramework="net46" />
<package id="MongoDB.Driver.Core" version="2.0.1" targetFramework="net46" />
<package id="NServiceKit.Common" version="1.0.31" targetFramework="net46" />
<package id="NServiceKit.Redis" version="1.0.16" targetFramework="net46" />
<package id="NServiceKit.Text" version="1.0.10" targetFramework="net46" />