Skip to content

Commit

Permalink
Add performance benchmarking files
Browse files Browse the repository at this point in the history
  • Loading branch information
@dhadka
dhadka committed Jul 31, 2024
1 parent e8dfd81 commit 51ea39f
Show file tree
Hide file tree
Showing 47 changed files with 2,476 additions and 105 deletions.
8 changes: 6 additions & 2 deletions .classpath
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,21 @@
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="test" value="true"/>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
<attribute name="test" value="true"/>
<attribute name="maven.pomderived" value="true"/>
<attribute name="optional" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-17">
<attributes>
<attribute name="module" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
Expand All @@ -36,5 +37,8 @@
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="lib" path="perf/native/NativeC/NativeC.jar"/>
<classpathentry kind="lib" path="perf/native/NativeCPP/NativeCPP.jar"/>
<classpathentry kind="lib" path="perf/native/NativeFortran/NativeFortran.jar"/>
<classpathentry kind="output" path="target/classes"/>
</classpath>
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ on:
- '19'
- '20'
- '21'
- '22'
default: '17'

env:
Expand Down
73 changes: 73 additions & 0 deletions .github/workflows/perf.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
name: Performance

on:
push:
paths-ignore:
- 'Timings.md'
workflow_dispatch:
inputs:
samples:
type: number
required: true
default: '10'
nfe:
type: number
required: true
default: '100000'
java-distribution:
type: choice
required: true
description: Java distribution
options:
- 'zulu'
- 'temurin'
- 'oracle'
default: 'zulu'
java-version:
type: choice
required: true
description: Java version
options:
- '17'
- '18'
- '19'
- '20'
- '21'
- '22'
default: '17'

env:
SAMPLES: ${{ inputs.samples || '10' }}
NFE: ${{ inputs.nfe || '100000' }}
JAVA_DISTRIBUTION: ${{ inputs.java-distribution || 'zulu' }}
JAVA_VERSION: ${{ inputs.java-version || '17' }}

jobs:
test:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ windows-latest, ubuntu-latest, macos-13 ]
steps:
- uses: actions/checkout@v4
- name: Set up JDK ${{ env.JAVA_VERSION }}
uses: actions/setup-java@v4
with:
java-version: ${{ env.JAVA_VERSION }}
distribution: ${{ env.JAVA_DISTRIBUTION }}
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- uses: actions/setup-python@v5
with:
python-version: 'pypy3.10'
- uses: fortran-lang/setup-fortran@v1
if: runner.os == 'macOS'
with:
compiler: gcc
version: 13
- name: Make
run: make -C perf
- name: Run Benchmarks
run: make SAMPLES=${{ env.SAMPLES }} NFE=${{ env.NFE }} run
9 changes: 8 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
/.settings/
/target/
/pisa_binaries/
/pisa_binaries/
**/bin/
**/*.jar
**/*.class
**/*.exe
**/*.so
**/*.dll
**/*.mod
14 changes: 14 additions & 0 deletions COPYING
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
Copyright 2009-2023 David Hadka and other contributors. All rights reserved.

The MOEA Framework is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or (at your
option) any later version.

The MOEA Framework is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
License for more details.

You should have received a copy of the GNU Lesser General Public License
along with the MOEA Framework. If not, see <http://www.gnu.org/licenses/>.
24 changes: 24 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Configure the platform-specific settings
ifeq ($(OS),Windows_NT)
SEPARATOR := ;
else
SEPARATOR := :
endif

CLASSPATH := lib/*$(SEPARATOR)native/NativeC/*$(SEPARATOR)native/NativeCPP/*$(SEPARATOR)native/NativeFortran/*$(SEPARATOR)bin

# Configure benchmark settings
SAMPLES ?= 10
NFE ?= 100000

build:
make -C perf
mkdir -p bin
javac -d bin -classpath "$(CLASSPATH)" src/org/moeaframework/performance/*.java

run:
java -classpath "$(CLASSPATH)" org.moeaframework.performance.Benchmark $(SAMPLES) $(NFE)

clean:
make -C perf clean
rm -rf bin
10 changes: 0 additions & 10 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,3 @@

This repository contains integration tests and performance benchmarks for the
[MOEA Framework](https://github.com/MOEAFramework/MOEAFramework).

## Release Steps

1. Create and merge a PR for the new version. Be sure to update `build.properties`, `releaseNotes.md`, and `README.md`.
2. Trigger the [staging workflow](https://github.com/MOEAFramework/MOEAFramework/actions/workflows/staging.yml) to
stage the release artifacts.
3. Update `pom.xml` in this repository with the new version and verify all tests pass.
4. Release the Maven artifacts on http://oss.sontatype.org.
5. Publish the GitHub release.
6. Publish the Website update.
32 changes: 32 additions & 0 deletions docs/ReleaseSteps.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# Release Steps

To publish a new version of the MOEA Framework or related project:

1. Create and merge a PR for the new version. Be sure to update `build.properties`, `releaseNotes.md`, and `README.md`.
2. Trigger the [staging workflow](https://github.com/MOEAFramework/MOEAFramework/actions/workflows/staging.yml) to
stage the release artifacts.
3. Update `pom.xml` in this repository with the new version and verify all tests pass.
4. Release the Maven artifacts on http://oss.sontatype.org.
5. Publish the GitHub release.
6. Publish the Website update.

## GPG Signing Keys

The following generates a new signing key. These keys are valid for several years, but if expired or lost we can always generate new keys.

1. Generate a new GPG key:
```
gpg --gen-key
# Name: MOEAFramework
# Email: [email protected]
# Password: <passpharse>
```
2. Publish key to key server:
```
gpg --keyserver keyserver.ubuntu.com --send-keys <keyId>
```
3. Create base64 encoded version of the key
```
gpg --armor --export-secret-key <keyId> | base64
```
4. Update the GitHub Actions secrets
175 changes: 175 additions & 0 deletions docs/Timings.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
# Timings

## v4.1

### Windows

#### Zulu 17

Timer | Min | Mean | Max | Count
-------------------------- | -------- | -------- | -------- | -----
C (pure) | 0.046139 | 0.047998 | 0.054955 | 10
Java | 0.569630 | 0.622625 | 1.086610 | 10
NativeC | 0.810032 | 0.822912 | 0.886378 | 10
NativeCPP | 0.810752 | 0.818092 | 0.833502 | 10
NativeFortran | 0.607998 | 0.666434 | 0.704614 | 10
NativeCDirectMapping | 0.617554 | 0.620496 | 0.625800 | 10
NativeCPPDirectMapping | 0.617444 | 0.621063 | 0.625424 | 10
NativeFortranDirectMapping | 0.356022 | 0.438267 | 0.492152 | 10
C (stdio) | 4.913521 | 4.976538 | 5.305515 | 10
C (socket) | 6.142973 | 6.194979 | 6.420488 | 10
Python (stdio) | 6.157954 | 6.464561 | 8.979980 | 10
Python (socket) | 8.080425 | 8.158695 | 8.229466 | 10
Pypy (stdio) | 6.152674 | 6.196714 | 6.283287 | 10
Pypy (socket) | 8.106453 | 8.153983 | 8.192417 | 10
JMetal (Direct) | 1.429589 | 1.520398 | 1.947543 | 10
JMetal (Plugin) | 1.438056 | 1.467239 | 1.561400 | 10

#### Zulu 21

Timer | Min | Mean | Max | Count
-------------------------- | -------- | -------- | --------- | -----
C (pure) | 0.046535 | 0.048065 | 0.053867 | 10
Java | 0.641785 | 0.683492 | 1.031204 | 10
NativeC | 0.876318 | 0.889635 | 0.914675 | 10
NativeCPP | 0.876214 | 0.886878 | 0.906669 | 10
NativeFortran | 0.591497 | 0.615169 | 0.718097 | 10
NativeCDirectMapping | 0.688670 | 0.691265 | 0.694350 | 10
NativeCPPDirectMapping | 0.688413 | 0.691923 | 0.700653 | 10
NativeFortranDirectMapping | 0.401006 | 0.438141 | 0.665779 | 10
C (stdio) | 4.861310 | 4.927665 | 5.388354 | 10
C (socket) | 6.096818 | 6.139152 | 6.268109 | 10
Python (stdio) | 6.059604 | 7.576872 | 21.026262 | 10
Python (socket) | 8.049068 | 8.078600 | 8.110983 | 10
Pypy (stdio) | 6.047997 | 6.089479 | 6.139745 | 10
Pypy (socket) | 8.027765 | 8.078040 | 8.115483 | 10
JMetal (Direct) | 1.343439 | 1.436013 | 2.070550 | 10
JMetal (Plugin) | 1.343969 | 1.367598 | 1.504579 | 10

#### Temurin 21

Timer | Min | Mean | Max | Count
-------------------------- | -------- | -------- | --------- | -----
C (pure) | 0.046337 | 0.048049 | 0.054804 | 10
Java | 0.624161 | 0.687952 | 1.225983 | 10
NativeC | 0.857105 | 0.865403 | 0.929166 | 10
NativeCPP | 0.855618 | 0.861188 | 0.878808 | 10
NativeFortran | 0.626497 | 0.687632 | 0.843863 | 10
NativeCDirectMapping | 0.668965 | 0.670649 | 0.673683 | 10
NativeCPPDirectMapping | 0.669353 | 0.673662 | 0.681948 | 10
NativeFortranDirectMapping | 0.439724 | 0.479095 | 0.529250 | 10
C (stdio) | 5.011703 | 5.123594 | 5.609689 | 10
C (socket) | 5.538710 | 6.190895 | 6.281915 | 10
Python (stdio) | 6.279670 | 7.353158 | 16.284814 | 10
Python (socket) | 8.177021 | 8.268738 | 8.544111 | 10
Pypy (stdio) | 6.273967 | 6.370625 | 6.479000 | 10
Pypy (socket) | 8.144192 | 8.217022 | 8.279133 | 10
JMetal (Direct) | 1.371844 | 1.466468 | 2.076322 | 10
JMetal (Plugin) | 1.377282 | 1.401347 | 1.551327 | 10

#### Oracle 21

Timer | Min | Mean | Max | Count
-------------------------- | -------- | -------- | --------- | -----
C (pure) | 0.046450 | 0.048422 | 0.057255 | 10
Java | 0.622414 | 0.681340 | 1.189069 | 10
NativeC | 0.854916 | 0.866518 | 0.923084 | 10
NativeCPP | 0.853833 | 0.862489 | 0.915285 | 10
NativeFortran | 0.595279 | 0.663122 | 0.704248 | 10
NativeCDirectMapping | 0.667601 | 0.674632 | 0.717524 | 10
NativeCPPDirectMapping | 0.666744 | 0.673607 | 0.711223 | 10
NativeFortranDirectMapping | 0.410036 | 0.496597 | 0.568765 | 10
C (stdio) | 4.988605 | 5.116906 | 5.961952 | 10
C (socket) | 6.217381 | 6.313823 | 6.844034 | 10
Python (stdio) | 6.210051 | 7.879196 | 22.384712 | 10
Python (socket) | 8.112740 | 8.151765 | 8.230937 | 10
Pypy (stdio) | 6.226160 | 6.293262 | 6.524208 | 10
Pypy (socket) | 8.106160 | 8.153165 | 8.230393 | 10
JMetal (Direct) | 1.340921 | 1.435807 | 2.097271 | 10
JMetal (Plugin) | 1.345150 | 1.370353 | 1.511553 | 10

### Ubuntu

#### Zulu 17

Timer | Min | Mean | Max | Count
-------------------------- | -------- | -------- | -------- | -----
C (pure) | 0.012806 | 0.014835 | 0.023394 | 10
Java | 0.559984 | 0.598916 | 0.929259 | 10
NativeC | 0.759871 | 0.764972 | 0.789043 | 10
NativeCPP | 0.755922 | 0.761231 | 0.763657 | 10
NativeFortran | 0.617875 | 0.686885 | 0.996467 | 10
NativeCDirectMapping | 0.589447 | 0.591674 | 0.595298 | 10
NativeCPPDirectMapping | 0.586388 | 0.591569 | 0.600064 | 10
NativeFortranDirectMapping | 0.450105 | 0.488475 | 0.561178 | 10
C (stdio) | 3.598900 | 3.716310 | 3.985158 | 10
C (socket) | 5.224044 | 5.837958 | 6.273526 | 10
Python (stdio) | 4.253225 | 4.660105 | 7.079649 | 10
Python (socket) | 7.478937 | 7.617721 | 7.882981 | 10
Pypy (stdio) | 4.161933 | 4.359183 | 4.520943 | 10
Pypy (socket) | 7.367706 | 7.642398 | 7.976498 | 10
JMetal (Direct) | 1.447542 | 1.548280 | 2.085437 | 10
JMetal (Plugin) | 1.450485 | 1.476822 | 1.584241 | 10

#### Zulu 21

Timer | Min | Mean | Max | Count
-------------------------- | -------- | -------- | -------- | -----
C (pure) | 0.012864 | 0.014818 | 0.026081 | 10
Java | 0.581693 | 0.625879 | 1.010279 | 10
NativeC | 0.762620 | 0.767049 | 0.793534 | 10
NativeCPP | 0.761700 | 0.763914 | 0.770037 | 10
NativeFortran | 0.640578 | 0.659212 | 0.673166 | 10
NativeCDirectMapping | 0.609473 | 0.611265 | 0.614013 | 10
NativeCPPDirectMapping | 0.609397 | 0.611592 | 0.616648 | 10
NativeFortranDirectMapping | 0.466739 | 0.526775 | 0.850395 | 10
C (stdio) | 3.572528 | 3.686078 | 3.950719 | 10
C (socket) | 5.156382 | 5.859372 | 6.304854 | 10
Python (stdio) | 4.167667 | 4.691576 | 7.721009 | 10
Python (socket) | 7.177921 | 7.549979 | 7.772311 | 10
Pypy (stdio) | 4.153039 | 4.315621 | 4.505983 | 10
Pypy (socket) | 7.416520 | 7.608305 | 7.833790 | 10
JMetal (Direct) | 1.348897 | 1.417346 | 1.811658 | 10
JMetal (Plugin) | 1.356035 | 1.378036 | 1.481707 | 10

#### Temurin 21

Timer | Min | Mean | Max | Count
-------------------------- | -------- | -------- | -------- | -----
C (pure) | 0.012878 | 0.014892 | 0.025989 | 10
Java | 0.581598 | 0.624273 | 0.994254 | 10
NativeC | 0.762082 | 0.770921 | 0.819174 | 10
NativeCPP | 0.762483 | 0.767847 | 0.795892 | 10
NativeFortran | 0.644786 | 0.679457 | 0.725648 | 10
NativeCDirectMapping | 0.606698 | 0.609281 | 0.612008 | 10
NativeCPPDirectMapping | 0.607268 | 0.608640 | 0.611972 | 10
NativeFortranDirectMapping | 0.487290 | 0.507109 | 0.524941 | 10
C (stdio) | 3.605547 | 3.685573 | 3.895845 | 10
C (socket) | 5.099056 | 5.770733 | 6.252106 | 10
Python (stdio) | 4.168131 | 4.362569 | 4.484545 | 10
Python (socket) | 7.323747 | 7.577452 | 7.814459 | 10
Pypy (stdio) | 4.246756 | 4.335433 | 4.441734 | 10
Pypy (socket) | 7.274744 | 7.487126 | 7.661862 | 10
JMetal (Direct) | 1.342402 | 1.407190 | 1.769492 | 10
JMetal (Plugin) | 1.349553 | 1.369088 | 1.499520 | 10

#### Oracle 21

Timer | Min | Mean | Max | Count
-------------------------- | -------- | -------- | -------- | -----
C (pure) | 0.012779 | 0.014335 | 0.025659 | 10
Java | 0.578409 | 0.610714 | 0.885197 | 10
NativeC | 0.756190 | 0.763773 | 0.814525 | 10
NativeCPP | 0.754669 | 0.761874 | 0.800281 | 10
NativeFortran | 0.621404 | 0.645164 | 0.699636 | 10
NativeCDirectMapping | 0.605918 | 0.617742 | 0.667082 | 10
NativeCPPDirectMapping | 0.604621 | 0.612249 | 0.630454 | 10
NativeFortranDirectMapping | 0.465622 | 0.493220 | 0.562575 | 10
C (stdio) | 3.501958 | 3.620467 | 3.838786 | 10
C (socket) | 5.231016 | 6.067210 | 6.268623 | 10
Python (stdio) | 4.204344 | 4.820329 | 9.950621 | 10
Python (socket) | 7.185461 | 7.425337 | 7.581744 | 10
Pypy (stdio) | 4.219888 | 4.285537 | 4.396315 | 10
Pypy (socket) | 7.238517 | 7.404882 | 7.634558 | 10
JMetal (Direct) | 1.340301 | 1.431091 | 1.985724 | 10
JMetal (Plugin) | 1.348236 | 1.369588 | 1.480777 | 10
Loading

0 comments on commit 51ea39f

Please sign in to comment.