Abstract file access pattern for DSS provider

This is an intial change to abstract file access using
IPlateTilePyramid. This interface provides a single method that will
take a plate file name along with the desired level and coordinates and
will retrieve the associated image.

This change includes two implementations of this:

- ConfigurationManagerFilePlateTilePyramid allows for a request to a
  given plate file and returns a stream for the level and coordinates
  specified.
- AzurePlateTilePyramid retrieves the file from Azure blob storage

This new access method has been incorporated into the DSS.aspx page and
will surface the content via Azure or local storage via a configuration
flag (requires a restart of the service).

WWTMVC5/App_Start/UnityConfig.cs

@@ -1,7 +1,12 @@
 using System;
+using System.Configuration;
 using System.Linq;
+using Azure.Core;
+using Azure.Identity;
 using Microsoft.Practices.Unity;
 using WWT.Providers;
+using WWTWebservices;
+using WWTWebservices.Azure;
 
 namespace WWTMVC5
 {
@@ -34,6 +39,7 @@ public static IUnityContainer GetConfiguredContainer()
  public static void RegisterTypes(IUnityContainer container)
  {
  RegisterRequestProviders(container);
+ RegisterPlateFileProvider(container);
 
  // NOTE: To load from web.config uncomment the line below. Make sure to add a Microsoft.Practices.Unity.Configuration to the using statements.
  // container.LoadConfiguration();
@@ -52,5 +58,17 @@ private static void RegisterRequestProviders(IUnityContainer container)
  container.RegisterType(type);
  }
  }
+
+ private static void RegisterPlateFileProvider(IUnityContainer container)
+ {
+ if (ConfigReader<bool>.GetSetting("UseAzurePlateFiles"))
+ {
+ container.RegisterInstance<IPlateTilePyramid>(new AzurePlateTilePyramid(ConfigurationManager.AppSettings["AzurePlateFileContainer"], new DefaultAzureCredential()));
+ }
+ else
+ {
+ container.RegisterType<IPlateTilePyramid, ConfigurationManagerFilePlateTilePyramid>(new ContainerControlledLifetimeManager());
+ }
+ }
  }
 }

WWTMVC5/WWTMVC5.csproj

@@ -1377,6 +1377,10 @@
  <Project>{ee4a3106-572b-4ef4-9ab5-a22643309a57}</Project>
  <Name>WWT.Providers</Name>
  </ProjectReference>
+ <ProjectReference Include="..\WWTWebservices.Azure\WWTWebservices.Azure.csproj">
+ <Project>{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}</Project>
+ <Name>WWTWebservices.Azure</Name>
+ </ProjectReference>
  <ProjectReference Include="..\WWTWebservices\WWTWebservices.csproj">
  <Project>{1cf4d986-51ad-43f8-a84a-38b6ecba2172}</Project>
  <Name>WWTWebservices</Name>
@@ -1443,9 +1447,6 @@
  <PackageReference Include="Microsoft.IdentityModel">
  <Version>6.1.7600.16394</Version>
  </PackageReference>
- <PackageReference Include="Microsoft.WindowsAzure.ConfigurationManager">
- <Version>3.2.3</Version>
- </PackageReference>
  <PackageReference Include="Newtonsoft.Json">
  <Version>10.0.3</Version>
  </PackageReference>
@@ -1461,9 +1462,6 @@
  <PackageReference Include="WebGrease">
  <Version>1.6.0</Version>
  </PackageReference>
- <PackageReference Include="WindowsAzure.Storage">
- <Version>4.3.0</Version>
- </PackageReference>
  </ItemGroup>
  <PropertyGroup>
  <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">10.0</VisualStudioVersion>

WWTMVC5/Web.config

@@ -137,6 +137,10 @@ Content-Type: application/x-wt-->
  <add key="ResourcesVersion" value="5.3.65"/>
  <add key="HomePageCacheTimeout" value="5"/>
 
+ <!-- Azure storage access for plate files-->
+ <add key="UseAzurePlateFiles" value="true"/>
+ <add key="AzurePlateFileContainer" value="https://127.0.0.1:10000/devstoreaccount1"/>
+
  <!-- Overridden with Configuration Builders -->
  <add key="TourCache" value=""/>
  <add key="WWTTilesDir" value=""/>

WWTWebSiteOnly.sln

@@ -24,6 +24,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
  README.md = README.md
  EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WWTWebservices.Azure", "WWTWebservices.Azure\WWTWebservices.Azure.csproj", "{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}"
+EndProject
 Global
  GlobalSection(SolutionConfigurationPlatforms) = preSolution
  Debug|Any CPU = Debug|Any CPU
@@ -46,6 +48,10 @@ Global
  {31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Debug|Any CPU.Build.0 = Debug|Any CPU
  {31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Release|Any CPU.ActiveCfg = Release|Any CPU
  {31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Release|Any CPU.Build.0 = Release|Any CPU
+ {FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Release|Any CPU.Build.0 = Release|Any CPU
  EndGlobalSection
  GlobalSection(SolutionProperties) = preSolution
  HideSolutionNode = FALSE

WWTWebservices.Azure/AzurePlateTilePyramid.cs

@@ -0,0 +1,35 @@
+using Azure.Core;
+using Azure.Storage.Blobs;
+using System;
+using System.Collections.Concurrent;
+using System.IO;
+
+namespace WWTWebservices.Azure
+{
+ public class AzurePlateTilePyramid : IPlateTilePyramid
+ {
+ private readonly BlobServiceClient _service;
+ private readonly ConcurrentDictionary<string, BlobContainerClient> _containers;
+
+ public AzurePlateTilePyramid(string storageUri, TokenCredential credentials)
+ {
+ _service = new BlobServiceClient(new Uri(storageUri), credentials);
+ _containers = new ConcurrentDictionary<string, BlobContainerClient>();
+ }
+
+ public Stream GetStream(string plateName, int level, int x, int y)
+ {
+ var container = _containers.GetOrAdd(plateName, p =>
+ {
+ var name = Path.GetFileNameWithoutExtension(p).ToLowerInvariant();
+
+ return _service.GetBlobContainerClient(name);
+ });
+
+ var client = container.GetBlobClient($"L{level}X{x}Y{y}.png");
+ var download = client.Download();
+
+ return download.Value.Content;
+ }
+ }
+}

WWTWebservices.Azure/WWTWebservices.Azure.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+ <PropertyGroup>
+ <TargetFramework>net48</TargetFramework>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <PackageReference Include="Azure.Storage.Blobs" Version="12.6.0" />
+ </ItemGroup>
+
+ <ItemGroup>
+ <ProjectReference Include="..\WWTWebservices\WWTWebservices.csproj" />
+ </ItemGroup>
+
+ <ItemGroup>
+ <Reference Include="System.Configuration" />
+ </ItemGroup>
+
+</Project>

WWTWebservices/ConfigurationManagerFilePlateTilePyramid.cs

@@ -0,0 +1,69 @@
+using System.Collections;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Configuration;
+using System.IO;
+using System.Text.RegularExpressions;
+
+namespace WWTWebservices
+{
+ public class ConfigurationManagerFilePlateTilePyramid : IPlateTilePyramid
+ {
+ private readonly RegexDictionary _plateNamePath;
+
+ public ConfigurationManagerFilePlateTilePyramid()
+ {
+ _plateNamePath = new RegexDictionary
+ {
+ { @"dssterrapixel\.plate", "WWTTilesDir" },
+ { @"DSSpngL5to12_x(\d+)_y(\d+)\.plate", "DssTerapixelDir" },
+ };
+ }
+
+ public Stream GetStream(string plateName, int level, int x, int y)
+ {
+ var path = _plateNamePath.GetPath(plateName);
+
+ return PlateTilePyramid.GetFileStream(Path.Combine(path, plateName), level, x, y);
+ }
+
+ private class RegexDictionary : IEnumerable
+ {
+ private readonly Dictionary<Regex, string> _regex = new Dictionary<Regex, string>();
+ private readonly ConcurrentDictionary<string, string> _strings = new ConcurrentDictionary<string, string>();
+
+ public void Add(string pattern, string configName)
+ {
+ var path = ConfigurationManager.AppSettings[configName];
+
+ _regex.Add(new Regex(pattern, RegexOptions.Compiled), path);
+ }
+
+ IEnumerator IEnumerable.GetEnumerator() => _strings.GetEnumerator();
+
+ /// <summary>
+ /// Finds the directory of a platefile.
+ /// </summary>
+ /// <param name="plateFile">The platefile name</param>
+ /// <returns>The directory the platefile is in</returns>
+ /// <remarks>
+ /// This uses a list of regex entries that match a prospective platefile to
+ /// its known directory. Once identified, it is cached so no more regex
+ /// matches (which are potentially very expensive) must be performed to
+ /// identify its path.
+ /// </remarks>
+ public string GetPath(string plateFile) => _strings.GetOrAdd(plateFile, p =>
+ {
+ foreach (var t in _regex)
+ {
+ if (t.Key.IsMatch(plateFile))
+ {
+ return t.Value;
+ }
+ }
+
+ return null;
+ });
+ }
+ }
+}

WWTWebservices/IPlateTilePyramid.cs

@@ -0,0 +1,9 @@
+using System.IO;
+
+namespace WWTWebservices
+{
+ public interface IPlateTilePyramid
+ {
+ Stream GetStream(string plateName, int level, int x, int y);
+ }
+}

WWTWebservices/PlateTilePyramid.cs

@@ -238,14 +238,13 @@ public Stream GetFileStream(int level, int x, int y)
  static public Stream GetFileStream(string filename, int level, int x, int y)
  {
  uint offset = GetFileIndexOffset(level, x, y);
- uint length;
  uint start;
 
  MemoryStream ms = null;
  using (FileStream f = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
  {
  f.Seek(offset, SeekOrigin.Begin);
- start = GetNodeInfo(f, offset, out length);
+ start = GetNodeInfo(f, offset, out var length);
 
  byte[] buffer = new byte[length];
  f.Seek(start, SeekOrigin.Begin);

WWTWebservices/WWTWebservices.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk.WindowsDesktop">
  <PropertyGroup>
- <TargetFramework>net45</TargetFramework>
+ <TargetFramework>net48</TargetFramework>
  <OutputType>Library</OutputType>
  <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
  <UseWindowsForms>true</UseWindowsForms>

src/WWT.Providers/Providers/DSSProvider.cs

@@ -1,70 +1,58 @@
 using System;
-using System.Configuration;
 using System.IO;
 using WWTWebservices;
 
 namespace WWT.Providers
 {
  public class DSSProvider : RequestProvider
  {
- public override void Run(WwtContext context)
+ private readonly IPlateTilePyramid _plateTile;
+
+ public DSSProvider(IPlateTilePyramid plateTile)
  {
- string wwtTilesDir = ConfigurationManager.AppSettings["WWTTilesDir"];
-  string dssTerapixelDir = ConfigurationManager.AppSettings["DssTerapixelDir"];
+ _plateTile = plateTile;
+ }
 
- string query = context.Request.Params["Q"];
- string[] values = query.Split(',');
- int level = Convert.ToInt32(values[0]);
- int tileX = Convert.ToInt32(values[1]);
- int tileY = Convert.ToInt32(values[2]);
+ public override void Run(WwtContext context)
+ {
+ var (level, tileX, tileY) = context.GetLevelAndCoordinates();
 
- int octsetlevel = level;
- string filename;
+ using (var stream = GetStream(level, tileX, tileY))
+ {
+ if (stream is null)
+ {
+ context.Response.Write("No image");
+ context.Response.Close();
+ }
 
+ context.Response.ContentType = "image/png";
+ stream.CopyTo(context.Response.OutputStream);
+ context.Response.Flush();
+ context.Response.End();
+ }
+ }
 
+ private Stream GetStream(int level, int tileX, int tileY)
+ {
  if (level > 12)
  {
- context.Response.Write("No image");
- context.Response.Close();
- return;
+ return null;
  }
-
- if (level < 8)
+ else if (level < 8)
  {
- context.Response.ContentType = "image/png";
- Stream s = PlateTilePyramid.GetFileStream(wwtTilesDir + "\\dssterrapixel.plate", level, tileX, tileY);
- int length = (int)s.Length;
- byte[] data = new byte[length];
- s.Read(data, 0, length);
- context.Response.OutputStream.Write(data, 0, length);
- context.Response.Flush();
- context.Response.End();
- return;
+ return _plateTile.GetStream("dssterrapixel.plate", level, tileX, tileY);
  }
  else
  {
- int L = level;
- int X = tileX;
- int Y = tileY;
- string mime = "png";
- int powLev5Diff = (int)Math.Pow(2, L - 5);
- int X32 = X / powLev5Diff;
- int Y32 = Y / powLev5Diff;
- filename = string.Format(dssTerapixelDir + @"\DSS{0}L5to12_x{1}_y{2}.plate", mime, X32, Y32);
+ int L5 = level - 5;
+ int powLev5Diff = (int)Math.Pow(2, L5);
+ int X32 = tileX / powLev5Diff;
+ int Y32 = tileY / powLev5Diff;
 
- int L5 = L - 5;
- int X5 = X % powLev5Diff;
- int Y5 = Y % powLev5Diff;
- context.Response.ContentType = "image/png";
- Stream s = PlateTilePyramid.GetFileStream(filename, L5, X5, Y5);
- int length = (int)s.Length;
- byte[] data = new byte[length];
- s.Read(data, 0, length);
- context.Response.OutputStream.Write(data, 0, length);
- context.Response.Flush();
- context.Response.End();
- return;
+ int X5 = tileX % powLev5Diff;
+ int Y5 = tileY % powLev5Diff;
 
+ return _plateTile.GetStream($"DSSpngL5to12_x{X32}_y{Y32}.plate", L5, X5, Y5);
  }
  }
  }

src/WWT.Providers/QueryRequestProvider.cs

@@ -0,0 +1,18 @@
+using System;
+
+namespace WWT.Providers
+{
+ internal static class WwtContextExtensions
+ {
+ public static (int level, int tileX, int tileY) GetLevelAndCoordinates(this WwtContext context)
+ {
+ string query = context.Request.Params["Q"];
+ string[] values = query.Split(',');
+ int level = Convert.ToInt32(values[0]);
+ int tileX = Convert.ToInt32(values[1]);
+ int tileY = Convert.ToInt32(values[2]);
+
+ return (level, tileX, tileY);
+ }
+ }
+}