robot-tests: move robot tests from ci repo

openeventmachine · Aug 23, 2022 · ddbab44 · ddbab44
1 parent f3de2d2
commit ddbab44
Show file tree

Hide file tree

Showing 30 changed files with 1,048 additions and 18 deletions.
diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml
@@ -58,16 +58,10 @@ jobs:
         sudo apt-get update
         sudo apt-get install libconfig-dev
         sudo pip3 install robotframework
+        sudo apt-get install libcli-dev
+        sudo apt-get install telnet
     - name: Checkout EM-ODP
       uses: actions/checkout@v2
-    # Checkout ci for robot files
-    - name: Checkout ci
-      uses: actions/checkout@v2
-      with:
-        repository: ${{ github.repository_owner }}/ci
-        clean: false
-        path: ci
-        ref: master
     - name: Build EM-ODP
       run: ./scripts/build.sh
     - name: Run Robot Tests

diff --git a/robot-tests/README.md b/robot-tests/README.md
@@ -0,0 +1,16 @@
+# Nokia: Event Machine Robot Tests
+
+## Install robotframework in Ubuntu
+
+```bash
+sudo apt install python3-pip
+sudo pip3 install robotframework
+```
+
+## Manually run individual robot tests
+
+```bash
+robot --variable APPLICATION:<path_to_application>/<application> --variable CORE_MASK:<core_mask> --variable APPLICATION_MODE:<t/p> <path_to_robot_files>/<application>.robot
+e.g
+$ robot --variable APPLICATION:/home/username/EM/em-odp/build/programs/example/hello/hello --variable CORE_MASK:0xFE --variable APPLICATION_MODE:t /home/username/EM/em-odp/robot-tests/example/hello.robot
+```
diff --git a/robot-tests/common.resource b/robot-tests/common.resource
@@ -0,0 +1,94 @@
+*** Comments ***
+Copyright (c) 2020-2022, Nokia Solutions and Networks
+All rights reserved.
+SPDX-License-Identifier: BSD-3-Clause
+
+
+*** Settings ***
+Documentation    Resource file
+Library    Collections
+Library    Process
+Library    String
+
+
+*** Variables ***
+${KILL_TIMEOUT} =    30s
+
+# Match Pool Statistics
+@{POOL_STATISTICS_MATCH} =
+...    (buf|pkt)(\\s*[0-9]+)+(\\s*[0-3]+:\\[sz=[0-9]+\\s*n=[0-9]+\\(0/[0-9]+\\)\\s*\\$=[0-9]+\\])+
+
+# List of eligible return codes
+# The return code seem to depend on the version of the used robot framework.
+# Terminating a test with Ctrl-C sometimes returned 0, sometimes SIGINT(-2).
+# Timeout should return SIGKILL(-9) but not always. Since most of our programs
+# keeps running once started, our robot tests send SIGINT to the programs after
+# running them for a while or SIGKILL on timeout. So all of the return code 0,
+# -2(SIGINT) and -9(SIGKILL) can be considered as eligible.
+@{RC_LIST} =    ${0}    ${-2}    ${-9}
+
+# Regex pattern that must not be matched by all test cases
+@{REGEX_NOT_MATCH} =    EM ERROR
+
+# Common arguments used for all test cases
+@{CM_ARGS} =    -c    ${CORE_MASK}    -${APPLICATION_MODE}
+
+
+*** Keywords ***
+Kill Any Hanging Applications
+    [Documentation]    Kill any hanging applications
+    Terminate All Processes    kill=true
+
+Run EM-ODP Test
+    [Documentation]    Run em-odp application in the background, send SIGINT
+    ...    signal to the application process after sleeping the given sleep_time,
+    ...    then wait for the application to be completed or killed and check if
+    ...    the return code matches any of the eligible codes in RC_LIST and stdout
+    ...    matches given regex_match.
+
+    # sleep_time: sleep time (in seconds) before sending SIGINT to the em-odp application
+    # regex_match: regex pattern that must be matched
+    [Arguments]    ${sleep_time}    ${regex_match}
+
+    # In order to start application log from a new line
+    Log To Console    \n
+
+    # Run application with given arguments
+    ${app_handle} =    Process.Start Process    ${APPLICATION}
+    ...    @{CM_ARGS}
+    ...    stderr=STDOUT
+    ...    shell=True
+    ...    stdout=${TEMPDIR}/stdout.txt
+
+    IF    ${sleep_time}
+        # Sleep ${sleep_time} seconds
+        Sleep    ${sleep_time}
+        Send Signal To Process    SIGINT    ${app_handle}    group=true
+    END
+
+    ${output} =    Process.Wait For Process
+    ...    handle=${app_handle}
+    ...    timeout=${KILL_TIMEOUT}
+    ...    on_timeout=kill
+
+    # Log output
+    Log    ${output.stdout}    console=yes
+
+    # Verify the return code matches any of the eligible code in RC_LIST
+    List Should Contain Value
+    ...    ${RC_LIST}
+    ...    ${output.rc}
+    ...    Application Return Code: ${output.rc}
+
+    # Match regular expression lines
+    FOR    ${line}    IN    @{regex_match}
+        Should Match Regexp    ${output.stdout}    ${line}
+    END
+
+    FOR    ${line}    IN    @{REGEX_NOT_MATCH}
+        Should Not Match Regexp    ${output.stdout}    ${line}
+    END
+
+    FOR    ${line}    IN    @{POOL_STATISTICS_MATCH}
+        Should Match Regexp    ${output.stdout}    ${line}
+    END
diff --git a/robot-tests/example/api_hooks.robot b/robot-tests/example/api_hooks.robot
@@ -0,0 +1,34 @@
+*** Comments ***
+Copyright (c) 2020-2022, Nokia Solutions and Networks
+All rights reserved.
+SPDX-License-Identifier: BSD-3-Clause
+
+
+*** Settings ***
+Documentation     Test api_hooks -c ${CORE_MASK} -${APPLICATION_MODE}
+Resource          ../common.resource
+Test Setup        Set Log Level    TRACE
+Test Teardown     Kill Any Hanging Applications
+
+
+*** Variables ***
+# Regex pattern that must be matched by all the test cases
+@{REGEX_MATCH} =
+...    Dispatch enter callback
+...    Alloc-hook
+...    Free-hook
+...    Send-hook
+...    Done\\s*-\\s*exit
+
+# Regex pattern that must not be matched by all the test cases
+@{REGEX_NOT_MATCH} =
+...    EM ERROR
+...    failed!
+
+
+*** Test Cases ***
+Test api_hooks -c ${CORE_MASK} -${APPLICATION_MODE}
+    [Documentation]    api_hooks -c ${CORE_MASK} -${APPLICATION_MODE}
+    [Tags]    ${CORE_MASK}    ${APPLICATION_MODE}
+
+    Run EM-ODP Test    sleep_time=30    regex_match=${REGEX_MATCH}
diff --git a/robot-tests/example/dispatcher_callback.robot b/robot-tests/example/dispatcher_callback.robot
@@ -0,0 +1,37 @@
+*** Comments ***
+Copyright (c) 2020-2022, Nokia Solutions and Networks
+All rights reserved.
+SPDX-License-Identifier: BSD-3-Clause
+
+
+*** Settings ***
+Documentation    Test Dispatcher Callback -c ${CORE_MASK} -${APPLICATION_MODE}
+Resource    ../common.resource
+Test Setup        Set Log Level    TRACE
+Test Teardown     Kill Any Hanging Applications
+
+
+*** Variables ***
+# Common arguments used for all test cases
+${FIRST_REGEX} =    SEPARATOR=
+...    Dispatcher\\s*enter\\s*callback\\s*[1-2]+\\s*for\\s*EO:\\s*0x[a-fA-F0-9]+
+...    \\s*\\(EO\\s*[A-B]+\\)\\s*Queue:\\s*0x[a-fA-F0-9]+\\s*on\\s*core\\
+...    s*[0-9]+\\.\\s*Event\\s*seq:\\s*[0-9]+\\.
+
+${SECOND_REGEX} =    SEPARATOR=
+...    Ping\\s*from\\s*EO\\s*[A-B]+!\\s*Queue:\\s*0x[a-fA-f0-9]+\\s*on\\s*core
+...    \\s*[0-9]+\\.\\s*Event\\s*seq:\\s*[0-9]+\\.
+
+@{REGEX_MATCH} =
+...    ${FIRST_REGEX}
+...    ${SECOND_REGEX}
+...    Dispatcher\\s*exit\\s*callback\\s*[1-2]+\\s*for\\s*EO:\\s*0x[a-fA-f0-9]+
+...    Done\\s*-\\s*exit
+
+
+*** Test Cases ***
+Test Dispatcher Callback
+    [Documentation]    dispatcher_callback -c ${CORE_MASK} -${APPLICATION_MODE}
+    [TAGS]    ${CORE_MASK}    ${APPLICATION_MODE}
+
+    Run EM-ODP Test    sleep_time=30    regex_match=${REGEX_MATCH}
diff --git a/robot-tests/example/emcli.robot b/robot-tests/example/emcli.robot
@@ -0,0 +1,89 @@
+*** Comments ***
+Copyright (c) 2020-2022, Nokia Solutions and Networks
+All rights reserved.
+SPDX-License-Identifier: BSD-3-Clause
+
+
+*** Settings ***
+Documentation    Test EM CLI(Command Line Interface) with hello application
+Resource    ../common.resource
+Library           Telnet
+Test Setup        Set Log Level    TRACE
+Test Teardown     Kill Any Hanging Applications
+
+
+*** Variables ***
+@{REGEX_MATCH} =
+...    cli\\.enable: true\\(1\\)
+...    Starting CLI server on 127\\.0\\.0\\.1:55555
+...    CLI server terminated!
+...    Done\\s*-\\s*exit
+
+@{TELNET_REGEX} =
+...    Commands available:
+...    call\\s+odp_cls_print_all
+...    em_info_print\\s+[Name: all|cpu_arch|conf|help]
+...    em_core_print\\s+[Name: map|help]
+
+
+*** Test Cases ***
+Test Emcli
+    [Documentation]    hello -c ${CORE_MASK} -${APPLICATION_MODE}
+    [TAGS]    ${core_mask}    ${application_mode}
+
+    # In order to start application log from a new line
+    Log To Console    \n
+
+    # Run hello application with given arguments
+    ${app_handle} =    Process.Start Process    ${APPLICATION}
+    ...    @{CM_ARGS}
+    ...    stderr=STDOUT
+    ...    shell=True
+    ...    stdout=${TEMPDIR}/stdout.txt
+
+    Sleep    6s
+    Process Should Be Running    ${app_handle}
+
+    # Open telnet connection
+    Open Connection    localhost     port=55555
+    Write    help
+    # . normally will not match a newline, use (?s) to make . to match a newline
+    ${telnet_out} =    Read Until Regexp    (?s)EM-ODP>(.*?)EM-ODP>
+    Close All Connections
+
+    Send Signal To Process    SIGINT    ${app_handle}    group=true
+
+    ${output} =    Process.Wait For Process
+    ...    handle=${app_handle}
+    ...    timeout=${KILL_TIMEOUT}
+    ...    on_timeout=kill
+
+    # Log output
+    Log To Console    \n
+    Log    ${output.stdout}    console=yes
+    Log To Console    \nTelnet client output:\n    # To seperate the two logs
+    Log    ${telnet_out}    console=yes
+
+    # Verify the return code matches any of the eligible code in RC_LIST
+    List Should Contain Value
+    ...    ${RC_LIST}
+    ...    ${output.rc}
+    ...    Application Return Code: ${output.rc}
+
+    # Match telnet client outputs
+    FOR    ${line}    IN    @{TELNET_REGEX}
+        Should Match Regexp    ${telnet_out}    ${line}
+    END
+
+    # Match em-app outputs with REGEX_MATCH
+    FOR    ${line}    IN    @{REGEX_MATCH}
+        Should Match Regexp    ${output.stdout}    ${line}
+    END
+
+    FOR    ${line}    IN    @{REGEX_NOT_MATCH}
+        Should Not Match Regexp    ${output.stdout}    ${line}
+    END
+
+    FOR    ${line}    IN    @{POOL_STATISTICS_MATCH}
+        Should Match Regexp    ${output.stdout}    ${line}
+    END
diff --git a/robot-tests/example/error.robot b/robot-tests/example/error.robot
@@ -0,0 +1,34 @@
+*** Comments ***
+Copyright (c) 2020-2022, Nokia Solutions and Networks
+All rights reserved.
+SPDX-License-Identifier: BSD-3-Clause
+
+
+*** Settings ***
+Documentation    Test Error -c ${CORE_MASK} -${APPLICATION_MODE}
+Resource    ../common.resource
+Test Setup    Set Log Level    TRACE
+Test Teardown     Kill Any Hanging Applications
+
+
+*** Variables ***
+${FOUTH_REGEX} =    SEPARATOR=
+...    Appl\\s*EO\\s*specific\\s*error\\s*handler:\\s*EO\\s*0x[a-fA-Z0-9]+\\s*error
+...    \\s*0x[a-fA-Z0-9]+\\s*escope\\s*0x[a-fA-Z0-9]+
+
+@{REGEX_MATCH} =
+...    EM\\s*ERROR:0x[a-fA-Z0-9]+\\s*ESCOPE:0x[a-fA-Z0-9]+\\s*EO:0x[a-fA-Z0-9]+-"EO\\s*[A-fA-F]+"
+...    core:[0-9]+\\s*ecount:[0-9]+\\([0-9]+\\)\\s*event_machine_event.c:[0-9]+\\s*em_free\\(\\)
+...    Error\\s*log\\s*from\\s*EO\\s*[a-fA-Z]+\\s*\\[[0-9]+\\]\\s*on\\s*core\\s*[0-9]+!
+...    ${FOUTH_REGEX}
+...    Done\\s*-\\s*exit
+
+@{REGEX_NOT_MATCH} =    NO ERROR
+
+
+*** Test Cases ***
+Test Error
+    [Documentation]    error -c ${CORE_MASK} -${APPLICATION_MODE}
+    [TAGS]    ${CORE_MASK}    ${APPLICATION_MODE}
+
+    Run EM-ODP Test    sleep_time=25    regex_match=${REGEX_MATCH}
diff --git a/robot-tests/example/event_group.robot b/robot-tests/example/event_group.robot
@@ -0,0 +1,27 @@
+*** Comments ***
+Copyright (c) 2020-2022, Nokia Solutions and Networks
+All rights reserved.
+SPDX-License-Identifier: BSD-3-Clause
+
+
+*** Settings ***
+Documentation    Test Event Group -c ${CORE_MASK} -${APPLICATION_MODE}
+Resource    ../common.resource
+Test Setup        Set Log Level    TRACE
+Test Teardown     Kill Any Hanging Applications
+
+
+*** Variables ***
+@{REGEX_MATCH} =
+...    Start\\s*event\\s*group
+...    Event\\s*group\\s*notification\\s*event\\s*received\\s*after\\s*256\\s*data\\s*events\\.
+...    Cycles\\s*curr:[0-9]+,\\s*ave:[0-9]+
+...    Done\\s*-\\s*exit
+
+
+*** Test Cases ***
+Test Event Group
+    [Documentation]    event_group -c ${CORE_MASK} -${APPLICATION_MODE}
+    [TAGS]    ${CORE_MASK}    ${APPLICATION_MODE}
+
+    Run EM-ODP Test    sleep_time=30    regex_match=${REGEX_MATCH}
diff --git a/robot-tests/example/event_group_abort.robot b/robot-tests/example/event_group_abort.robot
@@ -0,0 +1,33 @@
+*** Comments ***
+Copyright (c) 2020-2022, Nokia Solutions and Networks
+All rights reserved.
+SPDX-License-Identifier: BSD-3-Clause
+
+
+*** Settings ***
+Documentation    Test Event Group Abort -c ${CORE_MASK} -${APPLICATION_MODE}
+Resource    ../common.resource
+Test Setup        Set Log Level    TRACE
+Test Teardown     Kill Any Hanging Applications
+
+
+*** Variables ***
+@{REGEX_MATCH}    Entering the event dispatch loop
+...    Round [0-9]+
+...    Created\\s*[0-9]+\\s*event\\s*group\\(s\\)\\s*with\\s*count\\s*of\\s*[0-9]+
+...    Abort\\s*group\\s*when\\s*received\\s*[0-9]+\\s*events
+...    Evgrp\\s*events:\\s*Valid:[0-9]+\\s*Expired:[0-9]+
+...    Evgrp\\s*increments:\\s*Valid:[0-9]+\\s*Failed:[0-9]+
+...    Evgrp\\s*assigns:\\s*Valid:[0-9]+\\s*Failed:[0-9]+
+...    Aborted\\s*[0-9]+\\s*event\\s*groups
+...    Failed\\s*to\\s*abort\\s*[0-9]+\\s*times
+...    Received\\s*[0-9]+\\s*notification\\s*events
+...    Freed\\s*[0-9]+\\s*notification\\s*events
+
+
+*** Test Cases ***
+Test Event Group Abort
+    [Documentation]    event_group_abort -c ${CORE_MASK} -${APPLICATION_MODE}
+    [TAGS]    ${CORE_MASK}    ${APPLICATION_MODE}
+
+    Run EM-ODP Test    sleep_time=30    regex_match=${REGEX_MATCH}