ExternalVendorCode/trick
1
0
Fork 0
mirror of https://github.com/nasa/trick.git synced 2024-12-18 12:56:26 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Variable Server Integration Test (#1406)

Browse Source 
* Added restart to trickops script; updated varserver integration test
This commit is contained in:
Jacqueline Deans 2022-12-05 15:24:20 -06:00 committed by GitHub
parent 3870dc73b6
commit 90b5cdcb3c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 708 additions and 448 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
test/SIM_test_varserv/RUN_test/realtime.py
View File

@ -1,20 +0,0 @@
import trick
def main():
trick.var_server_set_port(50000)
trick.set_var_server_info_msg_on()
#trick.sim_control_panel_set_enabled(True)
trick.real_time_enable()
trick.itimer_enable()
#trick.add_read(1.1, """vsx.vst.vs_read()""")
#trick.add_read(2.1, """vsx.vst.vs_read()""")
trick.exec_set_terminate_time(3000.0)
#print trick.var_add("vsx.vst.b")
if __name__ == "__main__":
main()

16
test/SIM_test_varserv/RUN_test/unit_test.py
View File

@ -7,16 +7,18 @@ def main():
trick.var_server_set_port(40000)
trick.var_ascii()
trick.real_time_enable()
trick.exec_set_software_frame(0.01)
# trick.set_var_server_info_msg_on()
#trick.set_var_server_info_msg_on()
trick.exec_set_terminate_time(100.0)
trick_utest.unit_tests.enable() ;
trick_utest.unit_tests.set_file_name( os.getenv("TRICK_HOME") + "/trick_test/SIM_test_varserv.xml" )
trick_utest.unit_tests.set_test_name( "VariableServerTest" )
varServerPort = trick.var_server_get_port()
test_output = ( os.getenv("TRICK_HOME") + "/trick_test/SIM_test_varserv.xml" )
command = 'os.system("./models/test_client/test_client ' + str(varServerPort) + ' --gtest_output=xml:' + test_output + ' &")'
TRICK_EXPECT_EQ(trick.var_server_get_port(), 40000, "VariableServerTest", "SetPortNumber")
trick.exec_set_terminate_time(3000.0)
# Start the test client after everything has been initialized (hopefully)
trick.add_read(1.0, command)
if __name__ == "__main__":
main()

12
test/SIM_test_varserv/S_define
View File

@ -3,11 +3,6 @@ PURPOSE: ( S_define )
LIBRARY DEPENDENCIES:
(
(varserv/src/VS.cpp)
(varserv/src/VS_init.cpp)
(varserv/src/VS_default_data.cpp)
(varserv/src/VS_commands.cpp)
(varserv/src/VS_tests.cpp)
(varserv/src/VS_shutdown.cpp)
)
*************************************************************/
@ -22,16 +17,9 @@ class testSimObject : public Trick::SimObject {
testSimObject() {
("default_data") vst.default_vars();
("initialization") vst.init();
("initialization") vst.testAddRemove();
("initialization") vst.testUnits();
("initialization") vst.testExists();
("initialization") vst.testPause();
("initialization") vst.testSendOnce();
("shutdown") vst.shutdown();
}
};
// Instantiations
testSimObject vsx;

10
test/SIM_test_varserv/S_overrides.mk
View File

@ -2,3 +2,13 @@
TRICK_CFLAGS += -I./models
TRICK_CXXFLAGS += -I./models
all: test_client
clean: clean_test_client
TEST_CLIENT_LIBS += -L${GTEST_HOME}/lib64 -L${GTEST_HOME}/lib -lgtest -lgtest_main -lpthread
test_client: models/test_client/test_client.cpp
cd models/test_client; $(TRICK_CXX) test_client.cpp -o test_client $(TEST_CLIENT_LIBS) -std=c++11
clean_test_client:
rm -f models/test_client/test_client

1
test/SIM_test_varserv/models/test_client/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
test_client

561
test/SIM_test_varserv/models/test_client/test_client.cpp Normal file
View File

@ -0,0 +1,561 @@
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/select.h>
#include <sys/time.h>
#include <netinet/in.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <string>
#include <sstream>
#include <iostream>
#include <functional>
#include <cmath>
#include <gtest/gtest.h>
#define SOCKET_BUF_SIZE 20480
class Socket {
public:
int max_retries = 10;
Socket() : _initialized(false) {}
int init(std::string hostname, int port) {
_hostname = hostname;
_port = port;
int tries = 0;
while ((_socket_fd = socket(AF_INET, SOCK_STREAM, 0)) < 0 && tries < max_retries) tries++;
if (_socket_fd < 0) {
std::cout << "Socket connection failed" << std::endl;
return -1;
}
struct sockaddr_in serv_addr;
serv_addr.sin_family = AF_INET;
serv_addr.sin_port = htons(port); // convert to weird network byte format
if(inet_pton(AF_INET, "127.0.0.1", &serv_addr.sin_addr)<=0) {
std::cout << "Invalid address/ Address not supported" << std::endl;
return -1;
}
tries = 0;
int connection_status;
while ((connection_status = connect(_socket_fd, (struct sockaddr *)&serv_addr, sizeof(serv_addr))) < 0 && tries < max_retries) tries++;
if (connection_status < 0) {
std::cout << "Connection failed" << std::endl;
return -1;
}
_initialized = true;
return 0;
}
int send (std::string message) {
// weird syntax I've never used before - since the send syscall that i'm trying to use is overloaded in this class,
// I have to append :: to the front of it so that the compiler knows to look in the global namespace
int success = ::send(_socket_fd, message.c_str(), message.size(), 0);
if (success < message.size()) {
std::cout << "Failed to send message" << std::endl;
}
return success;
}
int operator<< (std::string message) {
return send(message);
}
std::string receive () {
char buffer[SOCKET_BUF_SIZE];
int numBytes = recv(_socket_fd, buffer, SOCKET_BUF_SIZE, 0);
if (numBytes < 0) {
} else if (numBytes < SOCKET_BUF_SIZE) {
buffer[numBytes] = '\0';
}
return std::string(buffer);
}
void operator>> (std::string& ret) {
ret = receive();
}
bool check_for_message_availible(long timeout_sec = 2) {
fd_set read_fd_set;
struct timeval timeout = { .tv_sec = timeout_sec, .tv_usec = 0 };
FD_ZERO(&read_fd_set);
FD_SET(_socket_fd, &read_fd_set);
// I have one question for the designers of the interface for this syscall: why
select(_socket_fd+1, &read_fd_set, NULL, NULL, &timeout);
return FD_ISSET(_socket_fd, &read_fd_set);
}
void clear_buffered_data() {
// Poll the socket
if (check_for_message_availible(0)) {
// Receive into the void if there is a message
receive();
}
// otherwise no worries
}
void close() {
::close(_socket_fd);
}
private:
int _port;
std::string _hostname;
int _socket_fd;
bool _initialized;
};
int strcmp_IgnoringWhiteSpace(std::string s1_str, std::string s2_str) {
const char * s1 = s1_str.c_str();
const char * s2 = s2_str.c_str();
int i1 = 0;
int i2 = 0;
while (1) {
while ( !isgraph( s1[i1] ) && s1[i1] != '\0') { i1++; }
while ( !isgraph( s2[i2] ) && s2[i2] != '\0') { i2++; }
if ( s1[i1] == '\0' && s2[i2] == '\0') { return 0; }
if ( s1[i1] != s2[i2]) {
if (s1[i1] < s2[i2]) {
return -1;
} else {
return 1;
}
}
i1++; i2++;
}
}
class VariableServerTest : public ::testing::Test {
protected:
VariableServerTest() {
socket_status = socket.init("localhost", 40000);
if (socket_status == 0) {
std::stringstream request;
request << "trick.var_set_client_tag(\"VSTest";
request << numSession++;
request << "\") \n";
socket << request.str();
}
}
~VariableServerTest() {
socket << "trick.var_exit()\n";
socket.close();
}
Socket socket;
int socket_status;
static int numSession;
};
int VariableServerTest::numSession = 0;
TEST_F (VariableServerTest, Strings) {
if (socket_status != 0) {
FAIL();
}
std::string reply;
socket << "trick.var_send_once(\"vsx.vst.o\")\n";
socket >> reply;
std::string expected("5\tYou will rejoice to hear that no disaster has accompanied the commencement of an enterprise which you have regarded with such evil forebodings. I arrived here yesterday, and my first task is to assure my dear sister of my welfare and increasing confidence in the success of my undertaking.");
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
expected = std::string("5\tI am already far north of London, and as I walk in the streets of Petersburgh, I feel a cold northern breeze play upon my cheeks, which braces my nerves and fills me with delight. Do you understand this feeling?");
socket << "trick.var_send_once(\"vsx.vst.p\")\n";
socket >> reply;
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
// TODO: Does wchar actually work?
// expected = std::string("5\tThis breeze, which has travelled from the regions towards which I am advancing, gives me a foretaste of those icy climes. Inspirited by this wind of promise, my daydreams become more fervent and vivid.");
// socket << "trick.var_send_once(\"vsx.vst.q\")\n";
// socket >> reply;
// std::cout << "\tExpected: " << expected << "\n\tActual: " << reply << std::endl;
// EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
}
TEST_F (VariableServerTest, AddRemove) {
if (socket_status != 0) {
FAIL();
}
std::string reply;
std::string expected;
socket << "trick.var_add(\"vsx.vst.c\")\n";
socket >> reply;
expected = std::string("0 -1234");
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
socket >> reply;
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
socket << "trick.var_add(\"vsx.vst.m\")\n";
socket >> reply;
expected = std::string("0 -1234 1");
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
socket << "trick.var_remove(\"vsx.vst.m\")\n";
socket >> reply;
expected = std::string("0 -1234");
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
}
TEST_F (VariableServerTest, BadRefResponse) {
if (socket_status != 0) {
FAIL();
}
std::string reply;
std::string expected;
std::cerr << "The purpose of this test is to cause an error. Error messages are expected." << std::endl;
socket << "trick.var_send_once(\"vsx.vst.no_such_variable\")\n";
socket >> reply;
expected = std::string("5 BAD_REF");
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
}
TEST_F (VariableServerTest, Units) {
if (socket_status != 0) {
FAIL();
}
std::string reply;
std::string expected;
std::cerr << "The purpose of this test is to cause an error. Error messages are expected." << std::endl;
socket << "trick.var_add(\"vsx.vst.c\")\ntrick.var_units(\"vsx.vst.c\",\"g\")\n";
socket >> reply;
expected = std::string("0 -1234");
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
socket << "trick.var_add(\"vsx.vst.e\",\"m\")\n";
socket >> reply;
expected = std::string("0 -1234 -123456 {m}");
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
socket << "trick.var_units(\"vsx.vst.e\",\"ft\")\n";
socket >> reply;
expected = std::string("0 -1234 -405039 {ft}");
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
}
TEST_F (VariableServerTest, SendOnce) {
if (socket_status != 0) {
FAIL();
}
std::string reply;
std::string expected;
socket << "trick.var_send_once(\"vsx.vst.e\")\n";
socket >> reply;
expected = std::string("5 -123456");
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
// Check that nothing is sent cyclically
EXPECT_EQ(socket.check_for_message_availible(), false);
socket << "trick.var_send_once(\"vsx.vst.n[0], vsx.vst.n[1], vsx.vst.n[2],\", 3)\n";
socket >> reply;
expected = std::string("5 0 1 2");
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
std::cerr << "The purpose of this test is to cause an error. Error messages are expected." << std::endl;
socket << "trick.var_send_once(\"vsx.vst.n[0], vsx.vst.n[1], vsx.vst.n[2],\", 4)\n";
// Wrong number of variables, should not send a response but should see an error message from the Trick side
EXPECT_EQ(socket.check_for_message_availible(), false);
}
TEST_F (VariableServerTest, Exists) {
if (socket_status != 0) {
FAIL();
}
std::string reply;
std::string expected;
socket << "trick.var_exists(\"vsx.vst.e\")\n";
socket >> reply;
expected = std::string("1 1");
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
socket << "trick.var_exists(\"vsx.vst.z\")\n";
socket >> reply;
expected = std::string("1 0");
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
}
TEST_F (VariableServerTest, Cycle) {
if (socket_status != 0) {
FAIL();
}
double cycle = 1.0;
double tolerance = 0.01;
int num_cycles = 5;
// Challenge: no loops allowed
// I've been reading about lamdbas and when you have a hammer........
// Test: compare the differences in the returned sim time, make sure the difference
// between them are equal to what var_cycle has been set to within some tolerance
// Problem: setting a tolerance empirically is subject to fluctuactions in the environment,
// which could cause the test to fail even though everthing is functioning correctly
// For example, the original version of tolerances here passed in all the CI pipelines
// except Mac, which for some reason was returning much more variation in cycle time
// Instead, calculate the average cycle time for a few cycles, and make sure this
// value is closer to the current set cycle time than the previous set cycle time.
// This gives us less information before, we are basically only testing that
// var_cycle is actually changing the cycle time, instead of testing that the
// cycle time is being closely adhered to, but it shouldn't fail the pipeline unnecessarily
// And testing something is better than nothing
auto parse_message_for_sim_time = [](const std::string& message) {
// For this case the message will be
// 0\t<time> {s}\t
// We only care about <time>
std::stringstream message_stream(message);
std::string token;
std::getline(message_stream, token, '\t');
std::getline(message_stream, token, ' ');
return std::stod(token);
};
// Tail recursion, just for fun
std::function<void(int, double, std::vector<double>& )> record_cycle_times = [&] (int n_cycles, double last_sim_time, std::vector<double>& cycle_times) {
if (n_cycles <= 0)
return;
double sim_time = parse_message_for_sim_time(socket.receive());
cycle_times.push_back(sim_time - last_sim_time);
record_cycle_times(n_cycles-1, sim_time, cycle_times);
};
// Does this count as tail recursion if the last thing is technically a return instead of a call to sum_vec?
std::function<double(std::vector<double>&, size_t)> sum_vec = [&] (std::vector<double>& vec, size_t index) -> double {
if (index == vec.size())
return 0;
return vec[index] + sum_vec(vec, index+1);
};
auto measure_cycle = [&] (double cycle_length, int iterations) -> double {
std::string command = "trick.var_cycle(" + std::to_string(cycle_length) + ")\n";
socket << command;
double sim_time = parse_message_for_sim_time(socket.receive());
std::vector<double> cycle_times;
record_cycle_times(iterations, sim_time, cycle_times);
return sum_vec(cycle_times, 0) / cycle_times.size();
};
auto closer_to = [] (double expected, double other, double test_val) -> bool {
return (fabs(expected - test_val)) < (fabs(other - test_val));
};
std::function<void(std::vector<double>&, size_t)> compare_cycle_times = [&] (std::vector<double>& test_cycle_times, size_t index) {
if (index == test_cycle_times.size())
return;
double measured_cycle_time = measure_cycle(test_cycle_times[index], 5);
EXPECT_TRUE(closer_to(test_cycle_times[index], test_cycle_times[index-1], measured_cycle_time)) << "Expected time: " << test_cycle_times[index] << " Actual time: " << measured_cycle_time;
compare_cycle_times(test_cycle_times, index+1);
};
std::string command = "trick.var_add(\"time\")\n";
socket << command;
std::vector<double> test_cycle_times = {0, 3.0, 0.1, 1.0, 0.5};
compare_cycle_times(test_cycle_times, 1);
}
TEST_F (VariableServerTest, Pause) {
if (socket_status != 0) {
FAIL();
}
std::string reply;
std::string expected;
socket << "trick.var_add(\"vsx.vst.f\")\ntrick.var_add(\"vsx.vst.i\")\ntrick.var_pause()\n";
// Make sure it's paused
EXPECT_EQ(socket.check_for_message_availible(), false);
socket << "trick.var_send()\n";
socket >> reply;
expected = std::string("0 123456 1234.5677");
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
// Make sure it doesn't send cyclically
EXPECT_EQ(socket.check_for_message_availible(), false);
socket << "trick.var_unpause()\n";
socket >> reply;
expected = std::string("0 123456 1234.5677");
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
}
TEST_F (VariableServerTest, CopyAndWriteModes) {
if (socket_status != 0) {
FAIL();
}
std::string reply;
std::string expected;
// We're just checking that every combination of modes is functional
// We can't test that they perform their copying and writing in the correct place from here
// Default is 0 0
socket << "trick.var_add(\"vsx.vst.a\")\ntrick.var_add(\"vsx.vst.b\")\n";
socket >> reply;
expected = "0 97 98";
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
// Clear out anything else that's been sent
// I may need to write something else for this
socket << "trick.var_pause()\n";
socket.clear_buffered_data();
// Copy mode 1 (VS_COPY_SCHEDULED) write mode 0 (VS_WRITE_ASYNC)
socket << "trick.var_clear()\ntrick.var_set_copy_mode(1)\ntrick.var_add(\"vsx.vst.c\")\ntrick.var_add(\"vsx.vst.d\")\ntrick.var_unpause()\n";
socket >> reply;
// With copy mode VS_COPY_SCHEDULED and write mode VS_WRITE_ASYNC, the first reply will be all 0 since the main time to copy has not occurred yet.
// Is this what we want? Maybe we should have more strict communication on whether the data has been staged so the first message isn't incorrect
// expected = "0 -1234 1234";
// EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
// std::cout << "\tExpected: " << expected << "\n\tActual: " << reply << std::endl;
socket >> reply;
expected = "0 -1234 1234";
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
// Clear out anything else that's been sent
socket << "trick.var_pause()\n";
socket.clear_buffered_data();
// Copy mode 1 (VS_COPY_SCHEDULED) write mode 1 (VS_WRITE_WHEN_COPIED)
socket << "trick.var_clear()\ntrick.var_set_write_mode(1)\ntrick.var_add(\"vsx.vst.e\")\ntrick.var_add(\"vsx.vst.f\")\ntrick.var_unpause()\n";
socket >> reply;
expected = "0 -123456 123456";
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
socket >> reply;
expected = "0 -123456 123456";
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
// Clear out anything else that's been sent
socket << "trick.var_pause()\n";
socket.clear_buffered_data();
// Copy mode 2 (VS_COPY_TOP_OF_FRAME) write mode 0 (VS_WRITE_ASYNC)
socket << "trick.var_clear()\ntrick.var_set_copy_mode(2)\ntrick.var_set_write_mode(0)\ntrick.var_add(\"vsx.vst.g\")\ntrick.var_add(\"vsx.vst.h\")\ntrick.var_unpause()\n";
// Same issue as copy mode 1 write mode 0
socket >> reply;
// expected = "0 -1234567 123456789";
// EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
// std::cout << "\tExpected: " << expected << "\n\tActual: " << reply << std::endl;
socket >> reply;
expected = "0 -1234567 123456789";
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
// Clear out anything else that's been sent
socket << "trick.var_pause()\n";
socket.clear_buffered_data();
// Copy mode 2 (VS_COPY_TOP_OF_FRAME) write mode 1 (VS_WRITE_WHEN_COPIED)
socket << "trick.var_clear()\ntrick.var_set_copy_mode(2)\ntrick.var_set_write_mode(1)\ntrick.var_add(\"vsx.vst.i\")\ntrick.var_add(\"vsx.vst.j\")\ntrick.var_unpause()\n";
socket >> reply;
expected = "0 1234.5677 -1234.56789";
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
socket >> reply;
expected = "0 1234.5677 -1234.56789";
EXPECT_EQ(strcmp_IgnoringWhiteSpace(reply, expected), 0);
// Clear out anything else that's been sent
socket << "trick.var_pause()\n";
socket.clear_buffered_data();
}
int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
int result = RUN_ALL_TESTS();
Socket socket;
socket.init("localhost", 40000);
if (result == 0) {
// Success
std::cout << "Exiting peacefully!" << std::endl;
socket << "vsx.vst.success() \n";
} else {
// Failure
std::cout << "Exiting with failure!" << std::endl;
socket << "vsx.vst.fail() \n";
}
socket << "trick.stop() \n";
return result;
}

38
test/SIM_test_varserv/models/varserv/include/VS.hh
View File

@ -2,10 +2,11 @@
PURPOSE: ( Variable server test )
REFERENCES: ( None )
ASSUMPTIONS AND LIMITATIONS: ( None )
PROGRAMMERS: ( (Lindsay Landry) (L3) (9-12-2013) )
PROGRAMMERS: ( (Lindsay Landry) (L3) (9-12-2013) ) ( (Jackie Deans) (CACI) (11-30-2022) )
********************************************************************************/
#include <wchar.h>
#include <string.h>
#include <string>
#include "trick_utils/comm/include/tc.h"
#include "trick_utils/comm/include/tc_proto.h"
@ -14,14 +15,6 @@ PROGRAMMERS: ( (Lindsay Landry) (L3) (9-12-2013) )
class VSTest {
public:
unsigned short port_num;
const char * hostest;
char got_read[80];
double cycle_rate;
TCDevice comm_device;
char a;
unsigned char b;
short c;
@ -36,28 +29,29 @@ class VSTest {
unsigned long long l;
bool m;
int n[5];
std::string o;
char * p;
wchar_t * q;
int status;
VSTest();
~VSTest();
int init();
int default_vars();
int vs_write(char* cmd_buffer);
int vs_read();
void printLine();
int shutdown();
// TEST FUNCTIONS
int testAddRemove();
int testExists();
int testPause();
int testSendOnce();
int testUnits();
int success();
int fail();
const char *status_messages[3] = {
"Variable Server Test Success",
"Variable Server Test Failure",
"Client failed to connect"
};
private:
int get_line(char* thing);
int strcmp_IgnoringWhiteSpace(const char* s1, const char* s2);
};
#endif

47
test/SIM_test_varserv/models/varserv/src/VS.cpp
View File

@ -4,10 +4,55 @@ REFERENCE: ( None )
ASSUMPTIONS AND LIMITATIONS: ( None )
CLASS: ( scheduled )
LIBRARY DEPENDENCY: ( VS.o )
PROGRAMMERS: ( (Lindsay Landry) (L3) (9-12-2013) )
PROGRAMMERS: ( (Lindsay Landry) (L3) (9-12-2013) (Jackie Deans) (CACI) (11-30-2022) )
*******************************************************************************/
#include <iostream>
#include "../include/VS.hh"
#include "trick/exec_proto.h"
#include <limits>
VSTest::VSTest() {}
VSTest::~VSTest() {}
int VSTest::default_vars() {
a = 'a'; //char
b = 'b'; //unsigned char
c = -1234; //short
d = 1234; //unsigned short
e = -123456; //int
f = 123456; //unsigned int
g = -1234567; //long
h = 123456789; //unsigned long
i = 1234.5678; //float
j = -1234.567890; //double
k = -12345678912345;//long long
l = 12345678912345; //unsigned long
m = true; //boolean
for (int i = 0; i < 5; i++) {
n[i] = i;
}
o = std::string("You will rejoice to hear that no disaster has accompanied the commencement of an enterprise which you have regarded with such evil forebodings. I arrived here yesterday, and my first task is to assure my dear sister of my welfare and increasing confidence in the success of my undertaking.");
p = "I am already far north of London, and as I walk in the streets of Petersburgh, I feel a cold northern breeze play upon my cheeks, which braces my nerves and fills me with delight. Do you understand this feeling?";
q = L"This breeze, which has travelled from the regions towards which I am advancing, gives me a foretaste of those icy climes. Inspirited by this wind of promise, my daydreams become more fervent and vivid.";
}
int VSTest::init() {
status = 2;
return (0);
}
int VSTest::fail() {
status = 1;
}
int VSTest::success() {
status = 0;
}
int VSTest::shutdown() {
std::cout << "Shutting down with status: " << status << " Message: " << status_messages[status] << std::endl;
exec_terminate_with_return( status , __FILE__ , __LINE__ , status_messages[status] ) ;
return(0);
}

77
test/SIM_test_varserv/models/varserv/src/VS_commands.cpp
View File

@ -1,77 +0,0 @@
/******************************TRICK HEADER*************************************
PURPOSE: ( Test variable server )
REFERENCE: ( None )
ASSUMPTIONS AND LIMITATIONS: ( None )
CLASS: ( scheduled )
LIBRARY DEPENDENCY: ( VS.o )
PROGRAMMERS: ( (Lindsay Landry) (L3) (9-12-2013) )
*******************************************************************************/
#include <iostream>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/resource.h>
#include "../include/VS.hh"
#include "sim_services/VariableServer/include/variable_server_proto.h"
#include "sim_services/VariableServer/include/VariableServer.hh"
// internal function: parse tc_read output
int VSTest::get_line(char* thing) {
int ii;
int count = 0;
// take one full line of tc device output
for (ii = 0; ii < strlen(thing); ii++) {
if (thing[ii] != '\n') {
got_read[ii] = thing[ii];
} else {
got_read[ii] = '\0';
break;
}
}
return(0);
}
void VSTest::printLine() {
std::cout << got_read << std::endl;
}
int VSTest::vs_write(char* cmd_buffer) {
int nbytes;
nbytes = strlen(cmd_buffer);
tc_write(&comm_device, cmd_buffer, nbytes);
return(0);
}
int VSTest::vs_read() {
int num;
double cpu_t, cpu_st;
char read_buffer[256];
struct rusage cpu_usg;
getrusage(RUSAGE_SELF, &cpu_usg);
cpu_st = ((double) cpu_usg.ru_utime.tv_sec) + ((double) cpu_usg.ru_utime.tv_usec/1000000.0);
while (true) {
getrusage(RUSAGE_SELF, &cpu_usg);
cpu_t = ((double) cpu_usg.ru_utime.tv_sec) + ((double) cpu_usg.ru_utime.tv_usec/1000000.0);
// Check for empty read.
if (cpu_t - cpu_st > 5.0*cycle_rate) {
memset(got_read, '\0', strlen(got_read));
break;
}
num = tc_pending(&comm_device);
if (num) {
tc_read(&comm_device, read_buffer, num);
get_line(read_buffer);
break;
}
}
return(0);
}

31
test/SIM_test_varserv/models/varserv/src/VS_default_data.cpp
View File

@ -1,31 +0,0 @@
/******************************TRICK HEADER*************************************
PURPOSE: ( Test variable server )
REFERENCE: ( None )
ASSUMPTIONS AND LIMITATIONS: ( None )
CLASS: ( scheduled )
LIBRARY DEPENDENCY: ( VS.o )
PROGRAMMERS: ( (Lindsay Landry) (L3) (9-12-2013) )
*******************************************************************************/
#include <iostream>
#include "../include/VS.hh"
#include "sim_services/VariableServer/include/variable_server_proto.h"
int VSTest::default_vars() {
a = 'a'; //char
b = 'b'; //unsigned char
c = -1234; //short
d = 1234; //unsigned short
e = -123456; //int
f = 123456; //unsigned int
g = -1234567; //long
h = 123456789; //unsigned long
i = 1234.5678; //float
j = -1234.567890; //double
k = -12345678912345;//long long
l = 12345678912345; //unsigned long
m = false; //boolean
for (int i = 0; i < 5; i++) {
n[i] = i;
}
}

44
test/SIM_test_varserv/models/varserv/src/VS_init.cpp
View File

@ -1,44 +0,0 @@
/******************************TRICK HEADER*************************************
PURPOSE: ( Test variable server )
REFERENCE: ( None )
ASSUMPTIONS AND LIMITATIONS: ( None )
CLASS: ( scheduled )
LIBRARY DEPENDENCY: ( VS.o )
PROGRAMMERS: ( (Lindsay Landry) (L3) (9-12-2013) )
*******************************************************************************/
#include <iostream>
#include "../include/VS.hh"
#include "sim_services/VariableServer/include/variable_server_proto.h"
#include "sim_services/VariableServer/include/VariableServer.hh"
int VSTest::init() {
char msg[256];
port_num = var_server_get_port();
hostest = "localhost";
memset(&comm_device, '\0', sizeof(TCDevice));
comm_device.hostname = const_cast<char*>(hostest);
comm_device.port = port_num;
//std::cout << bob.hostname << bob.port << std::endl;
comm_device.disable_handshaking = TC_COMM_TRUE;
comm_device.disabled = TC_COMM_FALSE;
tc_connect(&comm_device);
if ( tc_isValid (&comm_device) ) {
printf ("connection is valid\n");
} else {
printf ("connection is NOT valid\n");
}
// default cycle rate
cycle_rate = 0.01;
snprintf(msg, sizeof(msg), "trick.var_cycle(%.2f)\n", cycle_rate);
vs_write(msg);
return (0);
}

17
test/SIM_test_varserv/models/varserv/src/VS_shutdown.cpp
View File

@ -1,17 +0,0 @@
/******************************TRICK HEADER*************************************
PURPOSE: ( Test variable server )
REFERENCE: ( None )
ASSUMPTIONS AND LIMITATIONS: ( None )
CLASS: ( scheduled )
LIBRARY DEPENDENCY: ( VS.o )
PROGRAMMERS: ( (Lindsay Landry) (L3) (9-12-2013) )
*******************************************************************************/
#include <iostream>
#include "../include/VS.hh"
int VSTest::shutdown() {
tc_disconnect(&comm_device);
return(0);
}

204
test/SIM_test_varserv/models/varserv/src/VS_tests.cpp
View File

@ -1,204 +0,0 @@
/******************************TRICK HEADER*************************************
PURPOSE: ( Test variable server )
REFERENCE: ( None )
ASSUMPTIONS AND LIMITATIONS: ( None )
CLASS: ( scheduled )
LIBRARY DEPENDENCY: ( VS.o )
PROGRAMMERS: ( (Lindsay Landry) (L3) (9-12-2013) )
*******************************************************************************/
#include <sys/resource.h>
#include "../include/VS.hh"
#include "sim_services/VariableServer/include/variable_server_proto.h"
#include "sim_services/VariableServer/include/VariableServer.hh"
#include "sim_services/UnitTest/include/trick_tests.h"
int VSTest::strcmp_IgnoringWhiteSpace(const char* s1, const char* s2) {
int i1 = 0;
int i2 = 0;
while (1) {
while ( !isgraph( s1[i1] ) && s1[i1] != '\0') { i1++; }
while ( !isgraph( s2[i2] ) && s2[i2] != '\0') { i2++; }
if ( s1[i1] == '\0' && s2[i2] == '\0') { return 0; }
if ( s1[i1] != s2[i2]) {
if (s1[i1] < s2[i2]) {
return -1;
} else {
return 1;
}
}
i1++; i2++;
}
}
int VSTest::testUnits() {
char msg[256];
char suite[] = "VariableServerTest";
int result;
// INVALID UNIT CHANGE
snprintf(msg, sizeof(msg), "trick.var_add(\"vsx.vst.c\")\ntrick.var_units(\"vsx.vst.c\",\"g\")\n");
vs_write(msg);
std::cerr << "The purpose of this test is to cause an error. Error messages are expected." << std::endl;
vs_read();
result = strcmp_IgnoringWhiteSpace("0 -1234", got_read);
TRICK_EXPECT_EQ(result, 0, suite, "VariableInvalidUnits")
// ADD UNITS
snprintf(msg, sizeof(msg), "trick.var_add(\"vsx.vst.e\",\"m\")\n");
vs_write(msg);
vs_read();
result = strcmp_IgnoringWhiteSpace("0 -1234 -123456 {m}", got_read);
TRICK_EXPECT_EQ(result, 0, suite, "VariableAddUnits")
// CHANGE UNITS
snprintf(msg, sizeof(msg), "trick.var_units(\"vsx.vst.e\",\"ft\")\n");
vs_write(msg);
vs_read();
result = strcmp_IgnoringWhiteSpace("0 -1234 -405039 {ft}", got_read);
TRICK_EXPECT_EQ(result, 0, suite, "VariableChangeUnits")
// CLEAR VARIABLE SERVER
snprintf(msg, sizeof(msg), "trick.var_clear()\n");
vs_write(msg);
}
int VSTest::testAddRemove() {
char msg[256];
char suite[] = "VariableServerTest";
int result;
// NO UNITS
snprintf(msg, sizeof(msg), "trick.var_add(\"vsx.vst.c\")\n");
vs_write(msg);
vs_read();
std::cout << got_read << std::endl;
result = strcmp_IgnoringWhiteSpace("0 -1234", got_read);
TRICK_EXPECT_EQ(result, 0, suite, "VariableAddNoUnits")
vs_read();
result = strcmp_IgnoringWhiteSpace("0 -1234", got_read);
TRICK_EXPECT_EQ(result, 0, suite, "VariableAddCyclic")
// REMOVE SINGLE VARIABLE
snprintf(msg, sizeof(msg), "trick.var_remove(\"vsx.vst.e\")\n");
vs_write(msg);
vs_read();
result = strcmp_IgnoringWhiteSpace("0 -1234", got_read);
TRICK_EXPECT_EQ(result, 0, suite, "VariableRemove")
// CLEAR VARIABLE SERVER
snprintf(msg, sizeof(msg), "trick.var_clear()\n");
vs_write(msg);
vs_read();
TRICK_EXPECT_EQ(strcmp(got_read, ""), 0, suite, "VariableClear")
return(0);
}
int VSTest::testSendOnce() {
char msg[256];
char suite[] = "VariableServerTest";
int result;
// SEND ONCE
snprintf(msg, sizeof(msg), "trick.var_send_once(\"vsx.vst.e\")\n");
vs_write(msg);
vs_read();
result = strcmp_IgnoringWhiteSpace("5 -123456", got_read);
TRICK_EXPECT_EQ(result, 0, suite, "VariableSendOnce")
trick_test_add_parent( suite , "VariableSendOnce" , "");
// SEND ONCE LIST
snprintf(msg, sizeof(msg), "trick.var_send_once(\"vsx.vst.n[0], vsx.vst.n[1], vsx.vst.n[2],\", 3)\n");
vs_write(msg);
vs_read();
result = strcmp_IgnoringWhiteSpace("5 0 1 2", got_read);
TRICK_EXPECT_EQ(result, 0, suite, "VariableSendOnceList")
// SEND ONCE LIST - WRONG INDICES
snprintf(msg, sizeof(msg), "trick.var_send_once(\"vsx.vst.n[0], vsx.vst.n[1], vsx.vst.n[2],\", 4)\n");
vs_write(msg);
std::cerr << "The purpose of this test is to cause an error. Error messages are expected." << std::endl;
vs_read();
result = strcmp_IgnoringWhiteSpace("", got_read);
TRICK_EXPECT_EQ(result, 0, suite, "VariableSendOnceListError")
}
int VSTest::testExists() {
char msg[256];
char suite[] = "VariableServerTest";
int result;
// VARIABLE EXISTS
snprintf(msg, sizeof(msg), "trick.var_exists(\"vsx.vst.e\")\n");
vs_write(msg);
vs_read();
result = strcmp_IgnoringWhiteSpace("1 1",got_read);
TRICK_EXPECT_EQ(result, 0, suite, "VariableExists")
trick_test_add_parent( suite , "VariableExists" , "3587464751");
// VARIABLE DOES NOT EXIST
snprintf(msg, sizeof(msg), "trick.var_exists(\"vsx.vst.z\")\n");
vs_write(msg);
vs_read();
std::cout << "Check variable doesn't exist: " << got_read << std::endl;
result = strcmp_IgnoringWhiteSpace("1 0",got_read);
TRICK_EXPECT_EQ(result, 0, suite, "VariableNotExists")
snprintf(msg, sizeof(msg), "trick.var_clear()\n");
vs_write(msg);
return(0);
}
int VSTest::testPause() {
char msg[256];
char suite[] = "VariableServerTest";
int result;
snprintf(msg, sizeof(msg), "trick.var_add(\"vsx.vst.f\")\n");
vs_write(msg);
snprintf(msg, sizeof(msg), "trick.var_add(\"vsx.vst.i\")\n");
vs_write(msg);
// PAUSE
snprintf(msg, sizeof(msg), "trick.var_pause()\n");
vs_write(msg);
vs_read();
vs_read();
TRICK_EXPECT_EQ(strcmp(got_read, ""), 0, suite, "VariablePause")
trick_test_add_parent( suite , "VariablePause" , "964174074");
// SEND
snprintf(msg, sizeof(msg), "trick.var_send()\n");
vs_write(msg);
vs_read();
result = strcmp_IgnoringWhiteSpace("0 123456 1234.5677", got_read);
TRICK_EXPECT_EQ(result, 0, suite, "VariableSend")
trick_test_add_parent( suite , "VariableSend" , "67211805");
vs_read();
TRICK_EXPECT_EQ(strcmp(got_read, ""), 0, suite, "VariableSendNoCyclic")
trick_test_add_parent( suite , "VariableSendNoCyclic" , "67211805");
// UNPAUSE
snprintf(msg, sizeof(msg), "trick.var_unpause()\n");
vs_write(msg);
vs_read();
result = strcmp_IgnoringWhiteSpace("0 123456 1234.5677", got_read);
TRICK_EXPECT_EQ(result, 0, suite, "VariableUnpause")
trick_test_add_parent( suite , "VariableUnpause" , "964174074");
vs_read();
result = strcmp_IgnoringWhiteSpace("0 123456 1234.5677", got_read);
TRICK_EXPECT_EQ(result, 0, suite, "VariableUnpauseCyclic")
trick_test_add_parent( suite , "VariableUnpauseCyclic" , "964174074");
snprintf(msg, sizeof(msg), "trick.var_clear()\n");
vs_write(msg);
return(0);
}

30
test_sims.yml
View File

@ -1,5 +1,3 @@
# Compile only sims
SIM_alloc_test:
path: test/SIM_alloc_test
@ -149,13 +147,6 @@ SIM_ball_L3:
runs:
RUN_test/unit_test.py:
returns: 0
# SIM_amoeba:
# path: trick_sims/Cannon/SIM_amoeba
# build_command: "trick-CP -t"
# binary: "T_main_{cpu}_test.exe"
# runs:
# RUN_test/unit_test.py:
# returns: 0
SIM_cannon_aero:
path: trick_sims/Cannon/SIM_cannon_aero
build_command: "trick-CP -t"
@ -206,7 +197,6 @@ SIM_sun:
RUN_test/unit_test.py:
returns: 0
# Special cases
# setup.py dumps a checkpoint
@ -276,3 +266,23 @@ SIM_checkpoint_data_recording:
# returns: 0
# compare:
# - test/SIM_checkpoint_data_recording/RUN_test6/ref_log_foo2.csv vs. test/SIM_checkpoint_data_recording/RUN_test6/log_foo2.csv
# The variable server client and SIM_amoeba sometimes fail to connect and need to be retried
SIM_test_varserv:
path: test/SIM_test_varserv
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
labels:
- retries_allowed
runs:
RUN_test/unit_test.py:
returns: 0
SIM_amoeba:
path: trick_sims/Cannon/SIM_amoeba
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
labels:
- retries_allowed
runs:
RUN_test/unit_test.py:
returns: 0

48
trickops.py
View File

@ -6,6 +6,8 @@ sys.path.append(sys.argv[1] + "/share/trick/trickops")
from TrickWorkflow import *
from WorkflowCommon import Job
max_retries = 5
class SimTestWorkflow(TrickWorkflow):
def __init__( self, quiet, trick_top_level ):
# Create the trick_test directory if it doesn't already exist
@ -16,7 +18,7 @@ class SimTestWorkflow(TrickWorkflow):
num_cpus = os.cpu_count() if os.cpu_count() is not None else 8
TrickWorkflow.__init__(self, project_top_level=(trick_top_level), log_dir=(trick_top_level +'/trickops_logs/'),
trick_dir=trick_top_level, config_file=(trick_top_level + "/test_sims.yml"), cpus=num_cpus, quiet=quiet)
def run( self ):
build_jobs = self.get_jobs(kind='build')
@ -38,9 +40,31 @@ class SimTestWorkflow(TrickWorkflow):
first_phase_jobs.append(job)
run_jobs.remove(job)
# Some tests fail intermittently for reasons not related to the tests themselves, mostly network weirdness.
# Allow retries so that we can still cover some network-adjacent code
retry_allowed_sims = self.get_sims(labels='retries_allowed')
retry_allowed_jobs = [run.get_run_job() for run in [item for sublist in [sim.get_runs() for sim in retry_allowed_sims] for item in sublist]]
for job in retry_allowed_jobs:
run_jobs.remove(job)
builds_status = self.execute_jobs(build_jobs, max_concurrent=self.cpus, header='Executing all sim builds.')
first_phase_run_status = self.execute_jobs(first_phase_jobs, max_concurrent=self.cpus, header="Executing required first phase runs.")
runs_status = self.execute_jobs(run_jobs, max_concurrent=self.cpus, header='Executing all sim runs.')
# Run the retry_allowed jobs
self.execute_jobs(retry_allowed_jobs, max_concurrent=self.cpus, header='Executing retry-allowed runs.')
# If anything failed, try it again up to max_retries times
all_retried_status = 0
final_retry_jobs = []
for sim in retry_allowed_sims:
failing_runs = [run for run in sim.get_runs() if run.get_run_job().get_status() == Job.Status.FAILED]
for run in failing_runs:
status, final_job = self.retry_job(sim, run, max_retries)
final_retry_jobs += [final_job]
all_retried_status = all_retried_status or status
comparison_result = self.compare()
analysis_status = self.execute_jobs(analysis_jobs, max_concurrent=self.cpus, header='Executing all analysis.')
@ -48,7 +72,7 @@ class SimTestWorkflow(TrickWorkflow):
self.status_summary() # Print a Succinct summary
# Dump failing logs
jobs = build_jobs + first_phase_jobs + run_jobs
jobs = build_jobs + first_phase_jobs + run_jobs + final_retry_jobs
for job in jobs:
if job.get_status() == Job.Status.FAILED:
print("Failing job: ", job.name)
@ -56,7 +80,25 @@ class SimTestWorkflow(TrickWorkflow):
print(open(job.log_file, "r").read())
print ("*"*120, "\n")
return (builds_status or runs_status or first_phase_run_status or self.config_errors or comparison_result or analysis_status)
return (builds_status or runs_status or first_phase_run_status or all_retried_status or self.config_errors or comparison_result or analysis_status)
# Retries a job up to max_retries times and adds runs to the sim
# Returns tuple of (job_status, final retry job)
def retry_job(self, sim, run, max_retries):
tries = 0
job_failing = 1
retry_run = None
retry_job = None
while tries < max_retries and job_failing:
tries += 1
retry_run = TrickWorkflow.Run(sim_dir=run.sim_dir, input=run.input, binary=run.binary, returns=run.returns,log_dir=run.log_dir)
retry_job = retry_run.get_run_job()
retry_job.name = retry_job.name + "_retry_" + str(tries)
job_failing = self.execute_jobs([retry_job], max_concurrent=1, header="Retrying failed job")
sim.add_run(retry_run)
return (job_failing, retry_job)
if __name__ == "__main__":
should_be_quiet = os.getenv('CI') is not None