Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cut Windows exe file counts #1650

Merged
merged 9 commits into from
Oct 17, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,14 @@ jobs:
shell: bash
run: |
find ./Tools/rdmp/Databases.yaml -type f -exec sed -i 's/RDMP_/TEST_/g' {} \;
- name: BundleSource
shell: bash
run: |
mkdir -p Tools/BundleUpSourceIntoZip/output
rm -f Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip
echo "dir /s/b *.cs *.xml > srcbitsa.txt" | cmd
perl -pe '$_=reverse' < srcbitsa.txt | sort -t'\' -k1,1 -u | perl -pe '$_=reverse' > srcbits.txt
echo 7z a -mx=9 Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip @srcbits.txt | cmd
- name: Build
run: dotnet build --configuration Release --verbosity minimal
- name: Create MySql Logging, DQE and Cohort Building Cache Db
Expand Down Expand Up @@ -108,16 +116,9 @@ jobs:

- name: Package
run: |
dotnet publish Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj -r win-x64 --self-contained -c Release -o PublishWinForms -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal --nologo
dotnet publish Tools/rdmp/rdmp.csproj -r win-x64 --self-contained -c Release -o PublishWindows -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal --nologo
dotnet publish Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj -r win-x64 --self-contained -c Release -o PublishWinForms -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:IncludeNativeLibrariesForSelfExtract=true --verbosity minimal --nologo
dotnet publish Tools/rdmp/rdmp.csproj -r win-x64 --self-contained -c Release -o PublishWindows -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:IncludeNativeLibrariesForSelfExtract=true --verbosity minimal --nologo
dotnet publish Tools/rdmp/rdmp.csproj -r linux-x64 --self-contained -c Release -o PublishLinux -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal --nologo
- name: BundleSource
shell: bash
run: |
mkdir -p Tools/BundleUpSourceIntoZip/output
echo "dir /s/b *.cs *.xml > srcbitsa.txt" | cmd
perl -pe '$_=reverse' < srcbitsa.txt | sort -t'\' -k1,1 -u | perl -pe '$_=reverse' > srcbits.txt
echo 7z a -mx=9 Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip @srcbits.txt | cmd

- name: Sign
shell: bash
Expand All @@ -130,11 +131,11 @@ jobs:
mkdir -p dist
cmd /c wix\\build.cmd ${{ steps.version.outputs.rdmpversion }}
echo '"'$signtool'"' 'Sign /f GitHubActionsWorkflow.pfx /fd sha256 /tr http://timestamp.digicert.com /td sha256 /p ${{ secrets.DIGICERT_PASSWORD }} dist/rdmp.msi' | cmd
(cd PublishWindows ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-win-x64.zip . | cmd)
(cd PublishWindows ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-win-x64.zip rdmp.exe NLog.config *.yaml | cmd)
(cd PublishLinux ; echo 7z a -mx=0 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.zip . | cmd)
mv PublishLinux rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux
echo 7z a dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux | cmd
(cd PublishWinForms ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-client.zip . | cmd)
(cd PublishWinForms ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-client.zip ResearchDataManagementPlatform.exe | cmd)

- name: Install Perl dependencies
uses: shogo82148/[email protected]
Expand Down Expand Up @@ -178,6 +179,7 @@ jobs:
rm dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar

- name: Build Nuget packages
if: contains(github.ref, 'refs/tags/v')
shell: bash
run: |
for i in Rdmp.Core/Rdmp.Core.csproj Rdmp.UI/Rdmp.UI.csproj Tests.Common/Tests.Common.csproj
Expand Down
23 changes: 0 additions & 23 deletions Application/ResearchDataManagementPlatform/NLog.template.config

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,6 @@
<ItemGroup>
<EmbeddedResource Remove="Updates\UpdaterUI.resx" />
</ItemGroup>
<ItemGroup>
<None Remove="NLog.template.config" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="ConsoleControl" Version="1.3.0" />
<PackageReference Include="Autoupdater.NET.Official" Version="1.8.4" />
Expand Down Expand Up @@ -65,9 +62,6 @@
</Compile>
<Content Include="Icon\main.ico" />
<Content Include="Icon\main.png" />
<Content Include="NLog.template.config">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<EmbeddedResource Update="RDMPMainForm.resx">
<DependentUpon>RDMPMainForm.cs</DependentUpon>
</EmbeddedResource>
Expand Down
2 changes: 1 addition & 1 deletion Documentation/CodeTutorials/Coding.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ Each area of the RDMP codebase has its own documentation. These include:
- [Creating Right Click Context Menus](./CreatingANewRightClickMenu.md)
- [Double Click / Drag and drop](./DoubleClickAndDragDrop.md)
- [Writing Plugins](./PluginWriting.md)
- [Cohort Creation](./../../Rdmp.Core/CohortCreation/Readme.md)
- [Cohort Creation](./../../Rdmp.Core/CohortCreation/CohortCreation.md)
- [Command Line Interface (CLI)](./../../Rdmp.Core/CommandLine/Runners/ExecuteCommandRunner.md)

RDMP has over 20 class diagrams which you can open if you have visual studio. These files end in the extension `.cd`
Expand Down
14 changes: 7 additions & 7 deletions Documentation/CodeTutorials/FAQ.md
Original file line number Diff line number Diff line change
Expand Up @@ -334,12 +334,12 @@ Since each section is runnable independently it is trivially easy for RDMP to pr

![Cohort Builder Tree](./Images/CohortBuilderUI.png)

For full details on technical implementation see [Cohort Creation](../../Rdmp.Core/CohortCreation/Readme.md).
For full details on technical implementation see [Cohort Creation](../../Rdmp.Core/CohortCreation/CohortCreation.md).

## Does the Cohort Builder support Excel, CSV files etc?
<a name="cohort-build-with-file"/>

Yes. If a cohort is solely defined by the contents of such a file it can be [committed directly to the cohort store](../../Rdmp.Core/CohortCommitting/Readme.md).
Yes. If a cohort is solely defined by the contents of such a file it can be [committed directly to the cohort store](../../Rdmp.Core/CohortCommitting/CohortCommitting.md).

If you need to combine data in the file (or multiple files) with data in your database then the first step is to create a new Catalogue by uploading the data into your database. This has a number of advantages:

Expand Down Expand Up @@ -834,7 +834,7 @@ RDMP also supports creating the audit logging database in other [DBMS] types (e.
### System Logs

System logs can be enabled for RDMP. These are low level logs designed to be read by a software developer or IT specialist. To enable
this feature, locate the `NLog.template.config` file in your RDMP install directory. Rename the file `NLog.config` (i.e. remove the word template).
this feature, download [NLog.config](https://raw.githubusercontent.com/HicServices/RDMP/main/Tools/rdmp/NLog.config) to your RDMP install.
This template generates file logs to a `./logs/` sub-directory. You can adjust it to log anywhere including to databases/central server etc
by [following the NLog targets guide](https://nlog-project.org/config/). **Logs may contain identifiable or sensitive information so should
be secured appropriately if enabled**.
Expand Down Expand Up @@ -891,7 +891,7 @@ var catalogues = repository.GetAllObjects<Catalogue>();
var catalogueItems = repository.GetAllObjects<CatalogueItem>();
```

If you think the problem is more widespread then you can also use the [`IInjectKnown<T>`](./../../Reusable/MapsDirectlyToDatabaseTable/Injection/README.md) system to perform `Lazy` loads which prevents repeated calls to the same property going back to the database every time.
If you think the problem is more widespread then you can also use the [`IInjectKnown<T>`](./../../Rdmp.Core/MapsDirectlyToDatabaseTable/Injection/Injection.md) system to perform `Lazy` loads which prevents repeated calls to the same property going back to the database every time.



Expand All @@ -904,9 +904,9 @@ Yes there are over 1,000 unit and integration tests, this is covered in [Tests](
[hic_dataLoadRunID]: #hic_dataLoadRunID
[Data Load Engine]: #data-load-engine
[db_executor]: https://www.sqlmatters.com/Articles/Adding%20a%20db_executor%20role.aspx
[cohort databases]: ../../Rdmp.Core/CohortCommitting/Readme.md
[cohort database]: ../../Rdmp.Core/CohortCommitting/Readme.md
[query cache]: ../../Rdmp.Core/CohortCreation/Readme.md
[cohort databases]: ../../Rdmp.Core/CohortCommitting/CohortCommitting.md
[cohort database]: ../../Rdmp.Core/CohortCommitting/CohortCommitting.md
[query cache]: ../../Rdmp.Core/CohortCreation/CohortCreation.md
[UNION]: ./Glossary.md#UNION
[EXCEPT]: ./Glossary.md#EXCEPT
[INTERSECT]: ./Glossary.md#INTERSECT
Expand Down
4 changes: 2 additions & 2 deletions Documentation/CodeTutorials/Graphs.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ This will take you to the 'Edit' page where you can specify which dimensions to
The RDMP graphing engine supports Bar and Plot graphs. Not all [DBMS] implement all graph types. The following
table shows what is implemented:

| Graph Type | [Sql Server](https://github.com/HicServices/FAnsiSql/blob/main/Implementations/FAnsi.Implementations.MicrosoftSQL/README.md) | [MySql](https://github.com/HicServices/FAnsiSql/blob/main/Implementations/FAnsi.Implementations.MySql/README.md) | [Postgres](https://github.com/HicServices/FAnsiSql/blob/main/Implementations/FAnsi.Implementations.PostgreSql/README.md) | [Oracle](https://github.com/HicServices/FAnsiSql/blob/main/Implementations/FAnsi.Implementations.Oracle/README.md) |
| Graph Type | [Sql Server](https://github.com/HicServices/FAnsiSql/blob/main/FAnsiSql/Implementations/MicrosoftSQL/README.md) | [MySql](https://github.com/HicServices/FAnsiSql/blob/main/FAnsiSql/Implementations/MySql/README.md) | [Postgres](https://github.com/HicServices/FAnsiSql/blob/main/FAnsiSql/Implementations/PostgreSql/README.md) | [Oracle](https://github.com/HicServices/FAnsiSql/blob/main/FAnsiSql/Implementations/Oracle/README.md) |
|----|---|----|---|---|
| [Bar 1 Dimension](#bar-1-dimension) | yes | yes | yes | yes |
| [Bar 2 Dimensions](#bar-2-dimensions) | yes | yes | no | yes |
Expand Down Expand Up @@ -119,7 +119,7 @@ _Applying a TOP to PIVOT graphs limits the number of series in the PIVOT_

One of the core strengths of the RDMP graphing system is the ability to run graphs on Filters, Cohorts and/or [ExtractionConfiguration] datasets. This lets you rapidly confirm that the cohort you are building does not have data holes or missing trends.

To graph a cohort right click it and go to Graph. Make sure you have set up a [cohort caching database](../../Rdmp.Core/CohortCreation/Readme.md).
To graph a cohort right click it and go to Graph. Make sure you have set up a [cohort caching database](../../Rdmp.Core/CohortCreation/CohortCreation.md).

![Right clicking a cohort set in a CohortIdentificationConfiguration and graphing it](Images/Graphs/GraphCohort.png)

Expand Down
4 changes: 2 additions & 2 deletions Documentation/CodeTutorials/UserManual.md
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ There are a couple of other database types which can be created as and when you

|Database|Role|
|---|---|
| [Query Caching](./../../Rdmp.Core/CohortCreation/Readme.md)| Improves the performance of complex cohort identification configurations and anonymisation. Also allows cross server and plugin Cohort Builder elements (e.g. to [REST APIs](./FAQ.md#apis))|
| [Query Caching](./../../Rdmp.Core/CohortCreation/CohortCreation.md)| Improves the performance of complex cohort identification configurations and anonymisation. Also allows cross server and plugin Cohort Builder elements (e.g. to [REST APIs](./FAQ.md#apis))|
|Anonymisation|Provides a way of performing identifier dropping / substitution on data load for when you want an entirely anonymous data repository|
|[Plugin Databases](./FAQ.md#plugins)| RDMP supports plugins which can in some cases have their own database(s)|

Expand Down Expand Up @@ -585,7 +585,7 @@ New [CohortIdentificationConfigurations] are created with 3 containers:
- Inclusion Criteria
- Exclusion Criteria

The first container is an [EXCEPT] while the other two are [UNION]. For more information on how these work see the [Cohort Builder FAQ](./FAQ.md#cohort-builder-overview) or [Cohort Creation](../../Rdmp.Core/CohortCreation/Readme.md).
The first container is an [EXCEPT] while the other two are [UNION]. For more information on how these work see the [Cohort Builder FAQ](./FAQ.md#cohort-builder-overview) or [Cohort Creation](../../Rdmp.Core/CohortCreation/CohortCreation.md).

## Commit Cohort

Expand Down
8 changes: 4 additions & 4 deletions NoteForNewDevelopers.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,16 +40,16 @@ All technical and repo specific documentation are stored in markdown (`.md` form

** Performance **
- [Database Change Tracking (A Performance Enhancement)](./Documentation/CodeTutorials/ChangeTracking.md)
- [Reducing database calls with 'injection'](./Reusable/MapsDirectlyToDatabaseTable/Injection/README.md)
- [Reducing database calls with 'injection'](./Rdmp.Core/MapsDirectlyToDatabaseTable/Injection/Injection.md)

**Deep Dives**
- [How untyped CSV data is parsed by RDMP](./Documentation/CodeTutorials/CSVHandling.md)
- [How 'Bulk Insert' function works](./Documentation/CodeTutorials/DataTableUpload.md)
- [How xls / xlsx files are read by RDMP](./Documentation/CodeTutorials/ExcelHandling.md)
- [Multiple Linkage Columns (e.g. NHS Number or CHI)](./Documentation/CodeTutorials/MultipleExtractionIdentifiers.md)
- [Storing cohort lists](./Rdmp.Core/CohortCommitting/Readme.md)
- [Cohort Builder docs including info on list caching](./Rdmp.Core/CohortCreation/Readme.md)
- [Tree layout documentation](./Rdmp.Core/Providers/Readme.md)
- [Storing cohort lists](./Rdmp.Core/CohortCommitting/CohortCommitting.md)
- [Cohort Builder docs including info on list caching](./Rdmp.Core/CohortCreation/CohortCreation.md)
- [Tree layout documentation](./Rdmp.Core/Providers/Providers.md)
- [Aggregate Graphs](./Documentation/CodeTutorials/Graphs.md)
- [YamlRepository](./Documentation/CodeTutorials/YamlRepository.md)
- [Custom Metadata Reports](./Documentation/CodeTutorials/CustomMetadataSubstitutions.md)
Expand Down
2 changes: 1 addition & 1 deletion Rdmp.Core/CohortCommitting/CohortCommitting.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

A Cohort is a collection of unique person identifiers which can be linked against datasets during an extraction. This namespace covers saving a list of identifiers into a cohort database.

See [Cohort Creation](../CohortCreation/Readme.md) for building queries that identify cohorts from your database based on inclusion/exclusion criteria.
See [Cohort Creation](../CohortCreation/CohortCreation.md) for building queries that identify cohorts from your database based on inclusion/exclusion criteria.

## Cohort Storage

Expand Down
6 changes: 3 additions & 3 deletions Rdmp.Core/Rdmp.Core.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -320,9 +320,9 @@
<DotNetCliToolReference Include="Microsoft.XmlSerializer.Generator" Version="7.0.0" />
</ItemGroup>
<ItemGroup>
<Content Include="Curation\KeywordHelp.txt">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<EmbeddedResource Include="Curation\KeywordHelp.txt">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</EmbeddedResource>
</ItemGroup>
<ItemGroup>
<Folder Include="ReusableLibraryCode\" />
Expand Down
10 changes: 5 additions & 5 deletions Rdmp.Core/Repositories/Managers/CommentStoreWithKeywords.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,16 @@ namespace Rdmp.Core.Repositories.Managers;
/// <summary>
/// Subclass of <see cref="CommentStore"/> which also loads KeywordHelp.txt
/// </summary>
public class CommentStoreWithKeywords : CommentStore
public sealed class CommentStoreWithKeywords : CommentStore
{
public override void ReadComments(params string[] directoriesToLookInForComments)
{
base.ReadComments(directoriesToLookInForComments);

var keywords = new FileInfo("./Curation/KeywordHelp.txt");

if (keywords.Exists)
AddToHelp(File.ReadAllText(keywords.FullName));
var assembly=typeof(CommentStoreWithKeywords).Assembly;
using var stream = assembly.GetManifestResourceStream($"{assembly.GetName().Name}.Curation.KeywordHelp.txt");
using var reader = new StreamReader(stream ?? throw new ApplicationException("Unable to read KeywordHelp.txt resource"));
AddToHelp(reader.ReadToEnd());
}

private void AddToHelp(string keywordHelpFileContents)
Expand Down
4 changes: 2 additions & 2 deletions wix/build.cmd
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
@echo off
set RDMPVERSION=%1
set wix="C:\Program Files (x86)\WiX Toolset v3.11\bin"
cd /d wix
cd /d %~dp0
%wix%\candle.exe rdmp.wxs -dVersion=%RDMPVERSION% -arch x64 -ext WixUtilExtension -nologo
if errorlevel 1 exit 1
%wix%\light.exe rdmp.wixobj -ext WixUtilExtension -nologo
if errorlevel 1 exit 1
move rdmp.cli ..\dist\
move rdmp.msi ..\dist\
Loading