Skip to content

Commit

Permalink
change handling of EDSM json data to prevent out of memory error (jso…
Browse files Browse the repository at this point in the history 
…n data was 50MB, is now 1.4GB)
  • Loading branch information
@mhwlng
mhwlng committed Dec 12, 2020
1 parent 4c8ca6e commit 713fa11
Show file tree
Hide file tree
Showing 4 changed files with 71 additions and 44 deletions.
4 changes: 2 additions & 2 deletions Elite/Elite.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@
<Reference Include="Costura, Version=4.1.0.0, Culture=neutral, PublicKeyToken=9919ef960d84173d, processorArchitecture=MSIL">
<HintPath>..\packages\Costura.Fody.4.1.0\lib\net40\Costura.dll</HintPath>
</Reference>
<Reference Include="CsvHelper, Version=17.0.0.0, Culture=neutral, PublicKeyToken=8c4959082be5c823, processorArchitecture=MSIL">
<HintPath>..\packages\CsvHelper.17.0.1\lib\net47\CsvHelper.dll</HintPath>
<Reference Include="CsvHelper, Version=18.0.0.0, Culture=neutral, PublicKeyToken=8c4959082be5c823, processorArchitecture=MSIL">
<HintPath>..\packages\CsvHelper.18.0.0\lib\net47\CsvHelper.dll</HintPath>
</Reference>
<Reference Include="Hardcodet.Wpf.TaskbarNotification, Version=1.0.5.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\Hardcodet.NotifyIcon.Wpf.1.0.8\lib\net451\Hardcodet.Wpf.TaskbarNotification.dll</HintPath>
Expand Down
4 changes: 2 additions & 2 deletions Elite/Properties/AssemblyInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.5.8.0")]
[assembly: AssemblyFileVersion("1.5.8.0")]
[assembly: AssemblyVersion("1.5.9.0")]
[assembly: AssemblyFileVersion("1.5.9.0")]

[assembly: log4net.Config.XmlConfigurator(Watch = true)]
2 changes: 1 addition & 1 deletion Elite/packages.config
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Costura.Fody" version="4.1.0" targetFramework="net472" />
<package id="CsvHelper" version="17.0.1" targetFramework="net472" />
<package id="CsvHelper" version="18.0.0" targetFramework="net472" />
<package id="Hardcodet.NotifyIcon.Wpf" version="1.0.8" targetFramework="net472" />
<package id="HtmlRenderer.Core" version="1.5.0.6" targetFramework="net472" />
<package id="HtmlRenderer.WinForms" version="1.5.0.6" targetFramework="net472" />
Expand Down
105 changes: 66 additions & 39 deletions ImportData/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,34 +13,45 @@

namespace ImportData
{
class Program
public static class JsonReaderExtensions
{
private static readonly ILog Log =
LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
private static string GetExePath()
{
var strExeFilePath = System.Reflection.Assembly.GetExecutingAssembly().Location;
return Path.GetDirectoryName(strExeFilePath);
}

private static byte[] Decompress(byte[] gzip)
public static IEnumerable<T> ParseJson<T>(string path)
{
using (var stream = new GZipStream(new MemoryStream(gzip),
CompressionMode.Decompress))
path = Path.Combine(GetExePath(), path);

if (File.Exists(path))
{
const int size = 100000;
var buffer = new byte[size];
using (var memory = new MemoryStream())
var serializer = new JsonSerializer();
using (var s = File.Open(path, FileMode.Open))
{
int count;
do
using (var sr = new StreamReader(s))
{
count = stream.Read(buffer, 0, size);
if (count > 0)
using (var reader = new JsonTextReader(sr))
{
memory.Write(buffer, 0, count);
while (reader.Read())
{
if (reader.TokenType == JsonToken.StartObject)
{
yield return serializer.Deserialize<T>(reader);
}
}
}
}
while (count > 0);
return memory.ToArray();
}
}
}
}

class Program
{
private static readonly ILog Log =
LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);

private static string GetExePath()
{
Expand Down Expand Up @@ -136,32 +147,43 @@ private static bool NeedToUpdateFile(string path, int minutes)
return true;
}


private static string DownloadJson(string path, string url, ref bool wasUpdated)
private static void DownloadJson(string path, string url, ref bool wasUpdated)
{
path = Path.Combine(GetExePath(), path);

DeleteExpiredFile(path, 1440);

if (File.Exists(path))
{
return File.ReadAllText(path);
}
else
if (!File.Exists(path))
{
using (var client = new WebClient())
{
client.Headers[HttpRequestHeader.AcceptEncoding] = "gzip";
var data = client.DownloadData(url);
var decompress = Decompress(data);

File.WriteAllBytes(path, decompress);

wasUpdated = true;
using (var compressedStream = new MemoryStream(client.DownloadData(url)))
{
using (var stream = new GZipStream(compressedStream, CompressionMode.Decompress))
{
using (BinaryWriter binWriter =
new BinaryWriter(File.Open(path, FileMode.Create)))
{
const int size = 100000;
var buffer = new byte[size];
int count;
do
{
count = stream.Read(buffer, 0, size);
if (count > 0)
{
binWriter.Write(buffer, 0, count);
}
} while (count > 0);
}
}
}
}

return File.ReadAllText(path); // not efficient, but gets around out of memory error
wasUpdated = true;

}
}
}

Expand Down Expand Up @@ -636,30 +658,33 @@ static void Main(string[] args)
try
{
List<StationEDSM> stationsEDSM = null;
List<StationEDDB> stationsEDDBList = null;
Dictionary<string,StationEDDB> stationsEDDB;
List<PopulatedSystemEDDB> populatedSystemsEDDBList;
Dictionary<int, PopulatedSystemEDDB> populatedSystemsEDDBbyEdsmId;

var wasAnyUpdated = false;

Console.WriteLine("downloading populated systems from EDDB");

var jsonPopulatedsystemsEDDBText = DownloadJson(@"Data\populatedsystemsEDDB.json", "https://eddb.io/archive/v6/systems_populated.json", ref wasAnyUpdated);
DownloadJson(@"Data\populatedsystemsEDDB.json", "https://eddb.io/archive/v6/systems_populated.json", ref wasAnyUpdated);

Console.WriteLine("downloading station list from EDSM");

var jsonStationsEDSMText = DownloadJson(@"Data\stationsEDSM.json", "https://www.edsm.net/dump/stations.json.gz", ref wasAnyUpdated);
DownloadJson(@"Data\stationsEDSM.json", "https://www.edsm.net/dump/stations.json.gz", ref wasAnyUpdated);

Console.WriteLine("downloading station list from EDDB");

var jsonStationsEDDBText = DownloadJson(@"Data\stationsEDDB.json", "https://eddb.io/archive/v6/stations.json", ref wasAnyUpdated);
DownloadJson(@"Data\stationsEDDB.json", "https://eddb.io/archive/v6/stations.json", ref wasAnyUpdated);

Console.WriteLine("checking station and system data");

populatedSystemsEDDBList = JsonReaderExtensions.ParseJson<PopulatedSystemEDDB>(@"Data\populatedsystemsEDDB.json").ToList();

if (NeedToUpdateFile(@"Data\cnbsystems.json", 1440))
{
// there are multiple stations with the same name ???
var populatedSystemsEDDBbyName = JsonConvert
.DeserializeObject<List<PopulatedSystemEDDB>>(jsonPopulatedsystemsEDDBText)
var populatedSystemsEDDBbyName = populatedSystemsEDDBList
.GroupBy(x => x.Name).Select(x => x.First())
.ToDictionary(x => x.Name);

Expand All @@ -668,20 +693,22 @@ static void Main(string[] args)

if (wasAnyUpdated || NeedToUpdateFile(@"Data\painitestations.json", 15))
{
populatedSystemsEDDBbyEdsmId = JsonConvert
.DeserializeObject<List<PopulatedSystemEDDB>>(jsonPopulatedsystemsEDDBText)
populatedSystemsEDDBbyEdsmId = populatedSystemsEDDBList
.Where(x => x.EdsmId != null)
.ToDictionary(x => (int) x.EdsmId);

stationsEDSM = JsonConvert.DeserializeObject<List<StationEDSM>>(jsonStationsEDSMText);

stationsEDDBList = JsonReaderExtensions.ParseJson<StationEDDB>(@"Data\stationsEDDB.json").ToList();

// there are multiple stations with the same name ???
stationsEDDB = JsonConvert.DeserializeObject<List<StationEDDB>>(jsonStationsEDDBText)
stationsEDDB = stationsEDDBList
.GroupBy(x => x.Name).Select(x => x.First())
.ToDictionary(x => x.Name);

Console.WriteLine("looking up additional EDDB station information for all stations");

stationsEDSM = JsonReaderExtensions.ParseJson<StationEDSM>(@"Data\stationsEDSM.json").ToList();

stationsEDSM.ForEach(z =>
{
if (stationsEDDB.ContainsKey(z.Name))
Expand Down

0 comments on commit 713fa11

Please sign in to comment.