From 6e77b197c7667cc16d5e6216b0b07316a5692839 Mon Sep 17 00:00:00 2001 From: Todd Stellanova Date: Thu, 14 Mar 2019 21:09:47 -0700 Subject: [PATCH] Add DataValidatorGroup tests, add more DataValidator tests (#592) --- validation/tests/CMakeLists.txt | 15 +- validation/tests/test_data_validator.cpp | 253 ++++++---- .../tests/test_data_validator_group.cpp | 440 ++++++++++++++++++ validation/tests/tests_common.cpp | 103 ++++ validation/tests/tests_common.h | 68 +++ 5 files changed, 790 insertions(+), 89 deletions(-) create mode 100644 validation/tests/test_data_validator_group.cpp create mode 100644 validation/tests/tests_common.cpp create mode 100644 validation/tests/tests_common.h diff --git a/validation/tests/CMakeLists.txt b/validation/tests/CMakeLists.txt index e314cb38d8..d0540fa412 100644 --- a/validation/tests/CMakeLists.txt +++ b/validation/tests/CMakeLists.txt @@ -1,6 +1,6 @@ ############################################################################ # -# Copyright (c) 2015-2018 ECL Development Team. All rights reserved. +# Copyright (c) 2019 ECL Development Team. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -31,9 +31,16 @@ # ############################################################################ -add_executable(ecl_tests_data_validator test_data_validator.cpp) +add_executable(ecl_tests_data_validator test_data_validator.cpp tests_common.cpp) target_link_libraries(ecl_tests_data_validator ecl_validation) add_test(NAME ecl_tests_data_validator - COMMAND ecl_tests_data_validator -) + COMMAND ecl_tests_data_validator + ) + +add_executable(ecl_tests_data_validator_group test_data_validator_group.cpp tests_common.cpp) +target_link_libraries(ecl_tests_data_validator_group ecl_validation) + +add_test(NAME ecl_tests_data_validator_group + COMMAND ecl_tests_data_validator_group + ) diff --git a/validation/tests/test_data_validator.cpp b/validation/tests/test_data_validator.cpp index d3d9fb770c..620897bb5b 100644 --- a/validation/tests/test_data_validator.cpp +++ b/validation/tests/test_data_validator.cpp @@ -39,57 +39,18 @@ #include #include -//#include +#include #include -#include "../data_validator.h" +#include +#include -//void dump_validator_state(DataValidator* validator) -//{ -// uint32_t state = validator->state(); -// printf("state: 0x%x no_data: %d stale: %d timeout:%d\n", -// validator->state(), -// DataValidator::ERROR_FLAG_NO_DATA & state, -// DataValidator::ERROR_FLAG_STALE_DATA & state, -// DataValidator::ERROR_FLAG_TIMEOUT & state -// ); -// validator->print(); -//} - - -/** - * Insert a series of samples around a mean value - * @param validator The validator under test - * @param mean The mean value - * @param count The number of samples to insert in the validator - * @param rms_err (out) calculated rms error of the inserted samples - */ -void insert_values_around_mean(DataValidator *validator, const float mean, uint32_t count, float *rms_err) -{ - uint64_t timestamp = 500; - uint64_t timestamp_incr = 5; - const uint64_t error_count = 0; - const int priority = 50; - const float swing = 1E-2f; - double sum_dev_squares = 0.0f; - - //insert a series of values that swing around the mean - for (uint32_t i = 0; i < count; i++) { - float iter_swing = (0 == (i % 2)) ? swing : -swing; - float iter_val = mean + iter_swing; - float iter_dev = iter_val - mean; - sum_dev_squares += (iter_dev * iter_dev); - timestamp += timestamp_incr; - validator->put(timestamp, iter_val, error_count, priority); - } - - double rms = sqrt(sum_dev_squares / (double)count); - //note: this should be approximately equal to "swing" - *rms_err = (float)rms; -} void test_init() { + printf("\n--- test_init ---\n"); + uint64_t fake_timestamp = 666; + const uint32_t timeout_usec = 2000;//from original private value DataValidator *validator = new DataValidator; // initially there should be no siblings @@ -102,6 +63,9 @@ void test_init() assert(!validator->used()); // initially no priority assert(0 == validator->priority()); + validator->set_timeout(timeout_usec); + assert(validator->get_timeout() == timeout_usec); + DataValidator *sibling_validator = new DataValidator; validator->setSibling(sibling_validator); @@ -112,51 +76,59 @@ void test_init() uint32_t state = validator->state(); assert(DataValidator::ERROR_FLAG_NO_DATA == (DataValidator::ERROR_FLAG_NO_DATA & state)); + //verify that calling print doesn't crash tests + validator->print(); + + float *vibe_offset = validator->vibration_offset(); + assert(0.0f == vibe_offset[0]); + + delete validator; //force delete + } void test_put() { + printf("\n--- test_put ---\n"); + uint64_t timestamp = 500; - uint64_t timestamp_incr = 5; - const uint32_t timeout_usec = 2000;//from original private value + const uint32_t timeout_usec = 2000;//derived from class-private value float val = 3.14159f; - uint64_t error_count = 0; - int priority = 50; - //from private value: this is min change needed to avoid stale detection + //derived from class-private value: this is min change needed to avoid stale detection const float sufficient_incr_value = (1.1f * 1E-6f); - const int equal_value_count = 100; //default is private VALUE_EQUAL_COUNT_DEFAULT DataValidator *validator = new DataValidator; - validator->set_timeout(timeout_usec); - validator->set_equal_value_threshold(equal_value_count); - - //put a bunch of values that are all different - for (int i = 0; i < equal_value_count; i++, val += sufficient_incr_value) { - timestamp += timestamp_incr; - validator->put(timestamp, val, error_count, priority); - } + fill_validator_with_samples(validator, sufficient_incr_value, &val, ×tamp); assert(validator->used()); + //verify that the last value we inserted is the current validator value + float last_val = val - sufficient_incr_value; + assert(validator->value()[0] == last_val); // we've just provided a bunch of valid data: should be fully confident float conf = validator->confidence(timestamp); -// if (1.0f != conf) { -// printf("conf: %f\n",(double)conf); -// dump_validator_state(validator); -// } + + if (1.0f != conf) { + printf("conf: %f\n", (double)conf); + dump_validator_state(validator); + } + assert(1.0f == conf); // should be no errors assert(0 == validator->state()); //now check confidence much beyond the timeout window-- should timeout conf = validator->confidence(timestamp + (1.1 * timeout_usec)); -// if (0.0f != conf) { -// printf("conf: %f\n",(double)conf); -// dump_validator_state(validator); -// } + + if (0.0f != conf) { + printf("conf: %f\n", (double)conf); + dump_validator_state(validator); + } + assert(0.0f == conf); assert(DataValidator::ERROR_FLAG_TIMEOUT == (DataValidator::ERROR_FLAG_TIMEOUT & validator->state())); + delete validator; //force delete + } /** @@ -164,33 +136,30 @@ void test_put() */ void test_stale_detector() { + printf("\n--- test_stale_detector ---\n"); + uint64_t timestamp = 500; - uint64_t timestamp_incr = 5; float val = 3.14159f; - uint64_t error_count = 0; - int priority = 50; - const float insufficient_incr_value = (0.99 * 1E-6f);//insufficient to avoid stale detection - const int equal_value_count = 100; //default is private VALUE_EQUAL_COUNT_DEFAULT + //derived from class-private value, this is insufficient to avoid stale detection: + const float insufficient_incr_value = (0.99 * 1E-6f); DataValidator *validator = new DataValidator; - validator->set_equal_value_threshold(equal_value_count); - - //put a bunch of values that are all different - for (int i = 0; i < equal_value_count; i++, val += insufficient_incr_value) { - timestamp += timestamp_incr; - validator->put(timestamp, val, error_count, priority); - } + fill_validator_with_samples(validator, insufficient_incr_value, &val, ×tamp); // data is stale: should have no confidence assert(0.0f == validator->confidence(timestamp)); // should be a stale error uint32_t state = validator->state(); -// if (DataValidator::ERROR_FLAG_STALE_DATA != state) { -// dump_validator_state(validator); -// } + + if (DataValidator::ERROR_FLAG_STALE_DATA != state) { + dump_validator_state(validator); + } + assert(DataValidator::ERROR_FLAG_STALE_DATA == (DataValidator::ERROR_FLAG_STALE_DATA & state)); + delete validator; //force delete + } /** @@ -198,23 +167,134 @@ void test_stale_detector() */ void test_rms_calculation() { + printf("\n--- test_rms_calculation ---\n"); const int equal_value_count = 100; //default is private VALUE_EQUAL_COUNT_DEFAULT const float mean_value = 3.14159f; const uint32_t sample_count = 1000; float expected_rms_err = 0.0f; + uint64_t timestamp = 500; DataValidator *validator = new DataValidator; validator->set_equal_value_threshold(equal_value_count); - insert_values_around_mean(validator, mean_value, sample_count, &expected_rms_err); + insert_values_around_mean(validator, mean_value, sample_count, &expected_rms_err, ×tamp); float *rms = validator->rms(); assert(nullptr != rms); float calc_rms_err = rms[0]; float diff = fabsf(calc_rms_err - expected_rms_err); float diff_frac = (diff / expected_rms_err); -// printf("rms: %f expect: %f diff: %f frac: %f\n", (double)calc_rms_err, (double)expected_rms_err, -// (double)diff, (double)diff_frac); + printf("rms: %f expect: %f diff: %f frac: %f\n", (double)calc_rms_err, (double)expected_rms_err, + (double)diff, (double)diff_frac); assert(diff_frac < 0.03f); + + float *vibe_offset = validator->vibration_offset(); + float vibe_diff = fabsf(0.01005f - vibe_offset[0]); //TODO calculate this vibration value + printf("vibe: %f", (double)vibe_offset[0]); + assert(vibe_diff < 1E-3f); + + delete validator; //force delete + +} + +/** + * Verify error tracking performed by DataValidator::put + */ +void test_error_tracking() +{ + printf("\n--- test_error_tracking ---\n"); + uint64_t timestamp = 500; + uint64_t timestamp_incr = 5; + const uint32_t timeout_usec = 2000;//from original private value + float val = 3.14159f; + uint64_t error_count = 0; + int expected_error_density = 0; + int priority = 50; + //from private value: this is min change needed to avoid stale detection + const float sufficient_incr_value = (1.1f * 1E-6f); + //default is private VALUE_EQUAL_COUNT_DEFAULT + const int equal_value_count = 50000; + //should be less than equal_value_count: ensure this is less than NORETURN_ERRCOUNT + const int total_iterations = 1000; + + DataValidator *validator = new DataValidator; + validator->set_timeout(timeout_usec); + validator->set_equal_value_threshold(equal_value_count); + + //put a bunch of values that are all different + for (int i = 0; i < total_iterations; i++, val += sufficient_incr_value) { + timestamp += timestamp_incr; + + //up to a 50% random error rate appears to pass the error density filter + if ((((float)rand() / (float)RAND_MAX)) < 0.500f) { + error_count += 1; + expected_error_density += 1; + + } else if (expected_error_density > 0) { + expected_error_density -= 1; + } + + validator->put(timestamp, val, error_count, priority); + } + + assert(validator->used()); + //at this point, error_count should be less than NORETURN_ERRCOUNT + assert(validator->error_count() == error_count); + + // we've just provided a bunch of valid data with some errors: + // confidence should be reduced by the number of errors + float conf = validator->confidence(timestamp); + printf("error_count: %u validator confidence: %f\n", (uint32_t)error_count, (double)conf); + assert(1.0f != conf); //we should not be fully confident + assert(0.0f != conf); //neither should we be completely unconfident + // should be no errors, even if confidence is reduced, since we didn't exceed NORETURN_ERRCOUNT + assert(0 == validator->state()); + + // the error density will reduce the confidence by 1 - (error_density / ERROR_DENSITY_WINDOW) + // ERROR_DENSITY_WINDOW is currently private, but == 100.0f + float reduced_conf = 1.0f - ((float)expected_error_density / 100.0f); + double diff = fabs(reduced_conf - conf); + + if (reduced_conf != conf) { + printf("conf: %f reduced_conf: %f diff: %f\n", + (double)conf, (double)reduced_conf, diff); + dump_validator_state(validator); + } + + assert(diff < 1E-6f); + + //Now, insert a series of errors and ensure we trip the error detector + for (int i = 0; i < 250; i++, val += sufficient_incr_value) { + timestamp += timestamp_incr; + //100% error rate + error_count += 1; + expected_error_density += 1; + validator->put(timestamp, val, error_count, priority); + } + + conf = validator->confidence(timestamp); + assert(0.0f == conf); // should we be completely unconfident + // we should have triggered the high error density detector + assert(DataValidator::ERROR_FLAG_HIGH_ERRDENSITY == (DataValidator::ERROR_FLAG_HIGH_ERRDENSITY & validator->state())); + + + validator->reset_state(); + + //Now insert so many errors that we exceed private NORETURN_ERRCOUNT + for (int i = 0; i < 10000; i++, val += sufficient_incr_value) { + timestamp += timestamp_incr; + //100% error rate + error_count += 1; + expected_error_density += 1; + validator->put(timestamp, val, error_count, priority); + } + + conf = validator->confidence(timestamp); + assert(0.0f == conf); // should we be completely unconfident + // we should have triggered the high error count detector + assert(DataValidator::ERROR_FLAG_HIGH_ERRCOUNT == (DataValidator::ERROR_FLAG_HIGH_ERRCOUNT & validator->state())); + + delete validator; //force delete + } int main(int argc, char *argv[]) @@ -222,10 +302,13 @@ int main(int argc, char *argv[]) (void)argc; // unused (void)argv; // unused + srand(666); test_init(); test_put(); test_stale_detector(); test_rms_calculation(); + test_error_tracking(); + //TODO verify vibration calculation return 0; //passed } diff --git a/validation/tests/test_data_validator_group.cpp b/validation/tests/test_data_validator_group.cpp new file mode 100644 index 0000000000..f15fb19abd --- /dev/null +++ b/validation/tests/test_data_validator_group.cpp @@ -0,0 +1,440 @@ +/**************************************************************************** + * + * Copyright (c) 2019 Todd Stellanova. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ +/** + * @file test_data_validator_group.cpp + * Testing the DataValidatorGroup class + * + * @author Todd Stellanova + */ + +#include +#include +#include +#include +#include +#include +#include + + +const uint32_t base_timeout_usec = 2000;//from original private value +const int equal_value_count = 100; //default is private VALUE_EQUAL_COUNT_DEFAULT +const uint64_t base_timestamp = 666; +const unsigned base_num_siblings = 4; + + +/** + * Initialize a DataValidatorGroup with some common settings; + * @param sibling_count (out) the number of siblings initialized + */ +DataValidatorGroup *setup_base_group(unsigned *sibling_count) +{ + unsigned num_siblings = base_num_siblings; + + DataValidatorGroup *group = new DataValidatorGroup(num_siblings); + assert(nullptr != group); + //verify that calling print doesn't crash the tests + group->print(); + printf("\n"); + + //should be no failovers yet + assert(0 == group->failover_count()); + assert(DataValidator::ERROR_FLAG_NO_ERROR == group->failover_state()); + assert(-1 == group->failover_index()); + + //no vibration yet + float vibe_off = group->get_vibration_offset(base_timestamp, 0); + printf("vibe_off: %f \n", (double)vibe_off); + assert(-1.0f == group->get_vibration_offset(base_timestamp, 0)); + + float vibe_fact = group->get_vibration_factor(base_timestamp); + printf("vibe_fact: %f \n", (double)vibe_fact); + assert(0.0f == vibe_fact); + + //this sets the timeout on all current members of the group, as well as members added later + group->set_timeout(base_timeout_usec); + //the following sets the threshold on all CURRENT members of the group, but not any added later + //TODO This is likely a bug in DataValidatorGroup + group->set_equal_value_threshold(equal_value_count); + + //return values + *sibling_count = num_siblings; + + return group; + +} + +/** + * Fill one DataValidator with samples, by index. + * + * @param group + * @param val1_idx Index of the validator to fill with samples + * @param num_samples + */ +void fill_one_with_valid_data(DataValidatorGroup *group, int val1_idx, uint32_t num_samples) +{ + uint64_t timestamp = base_timestamp; + uint64_t error_count = 0; + float last_best_val = 0.0f; + + for (uint32_t i = 0; i < num_samples; i++) { + float val = ((float) rand() / (float) RAND_MAX); + float data[DataValidator::dimensions] = {val}; + group->put(val1_idx, timestamp, data, error_count, 100); + last_best_val = val; + } + + int best_idx = 0; + float *best_data = group->get_best(timestamp, &best_idx); + assert(last_best_val == best_data[0]); + assert(best_idx == val1_idx); +} + + + +/** + * Fill two validators in the group with samples, by index. + * Both validators will be filled with the same data, but + * the priority of the first validator will be higher than the second. + * + * @param group + * @param val1_idx index of the first validator to fill + * @param val2_idx index of the second validator to fill + * @param num_samples + */ +void fill_two_with_valid_data(DataValidatorGroup *group, int val1_idx, int val2_idx, uint32_t num_samples) +{ + uint64_t timestamp = base_timestamp; + uint64_t error_count = 0; + float last_best_val = 0.0f; + + for (uint32_t i = 0; i < num_samples; i++) { + float val = ((float) rand() / (float) RAND_MAX); + float data[DataValidator::dimensions] = {val}; + //two sensors with identical values, but different priorities + group->put(val1_idx, timestamp, data, error_count, 100); + group->put(val2_idx, timestamp, data, error_count, 10); + last_best_val = val; + } + + int best_idx = 0; + float *best_data = group->get_best(timestamp, &best_idx); + assert(last_best_val == best_data[0]); + assert(best_idx == val1_idx); + +} + +/** + * Dynamically add a validator to the group after construction + * @param group + * @return + */ +DataValidator *add_validator_to_group(DataValidatorGroup *group) +{ + DataValidator *validator = group->add_new_validator(); + //verify the previously set timeout applies to the new group member + assert(validator->get_timeout() == base_timeout_usec); + //for testing purposes, ensure this newly added member is consistent with the rest of the group + //TODO this is likely a bug in DataValidatorGroup + validator->set_equal_value_threshold(equal_value_count); + + return validator; +} + +/** + * Create a DataValidatorGroup and tack on two additional DataValidators + * + * @param validator1_handle (out) first DataValidator added to the group + * @param validator2_handle (out) second DataValidator added to the group + * @param sibling_count (in/out) in: number of initial siblings to create, out: total + * @return + */ +DataValidatorGroup *setup_group_with_two_validator_handles( + DataValidator **validator1_handle, + DataValidator **validator2_handle, + unsigned *sibling_count) +{ + DataValidatorGroup *group = setup_base_group(sibling_count); + + //now we add validators + *validator1_handle = add_validator_to_group(group); + *validator2_handle = add_validator_to_group(group); + *sibling_count += 2; + + return group; +} + + +void test_init() +{ + unsigned num_siblings = 0; + + DataValidatorGroup *group = setup_base_group(&num_siblings); + + //should not yet be any best value + int best_index = -1; + assert(nullptr == group->get_best(base_timestamp, &best_index)); + + delete group; //force cleanup +} + + +/** + * Happy path test of put method -- ensure the "best" sensor selected is the one with highest priority + */ +void test_put() +{ + unsigned num_siblings = 0; + DataValidator *validator1 = nullptr; + DataValidator *validator2 = nullptr; + + uint64_t timestamp = base_timestamp; + + DataValidatorGroup *group = setup_group_with_two_validator_handles(&validator1, &validator2, &num_siblings); + printf("num_siblings: %d \n", num_siblings); + unsigned val1_idx = num_siblings - 2; + unsigned val2_idx = num_siblings - 1; + + fill_two_with_valid_data(group, val1_idx, val2_idx, 500); + int best_idx = -1; + float *best_data = group->get_best(timestamp, &best_idx); + assert(nullptr != best_data); + float best_val = best_data[0]; + + float *cur_val1 = validator1->value(); + assert(nullptr != cur_val1); + //printf("cur_val1 %p \n", cur_val1); + assert(best_val == cur_val1[0]); + + float *cur_val2 = validator2->value(); + assert(nullptr != cur_val2); + //printf("cur_val12 %p \n", cur_val2); + assert(best_val == cur_val2[0]); + + delete group; //force cleanup +} + + +/** + * Verify that the DataValidatorGroup will select the sensor with the latest higher priority as "best". + */ +void test_priority_switch() +{ + unsigned num_siblings = 0; + DataValidator *validator1 = nullptr; + DataValidator *validator2 = nullptr; + + uint64_t timestamp = base_timestamp; + + DataValidatorGroup *group = setup_group_with_two_validator_handles(&validator1, &validator2, &num_siblings); + //printf("num_siblings: %d \n",num_siblings); + int val1_idx = (int)num_siblings - 2; + int val2_idx = (int)num_siblings - 1; + uint64_t error_count = 0; + + fill_two_with_valid_data(group, val1_idx, val2_idx, 100); + + int best_idx = -1; + float *best_data = nullptr; + //now, switch the priorities, which switches "best" but doesn't trigger a failover + float new_best_val = 3.14159f; + float data[DataValidator::dimensions] = {new_best_val}; + //a single sample insertion should be sufficient to trigger a priority switch + group->put(val1_idx, timestamp, data, error_count, 1); + group->put(val2_idx, timestamp, data, error_count, 100); + best_data = group->get_best(timestamp, &best_idx); + assert(new_best_val == best_data[0]); + //the new best sensor should now be the sensor with the higher priority + assert(best_idx == val2_idx); + //should not have detected a real failover + assert(0 == group->failover_count()); + + delete group; //cleanup +} + +/** + * Verify that the DataGroupValidator will prefer a sensor with no errors over a sensor with high errors + */ +void test_simple_failover() +{ + unsigned num_siblings = 0; + DataValidator *validator1 = nullptr; + DataValidator *validator2 = nullptr; + + uint64_t timestamp = base_timestamp; + + DataValidatorGroup *group = setup_group_with_two_validator_handles(&validator1, &validator2, &num_siblings); + //printf("num_siblings: %d \n",num_siblings); + int val1_idx = (int)num_siblings - 2; + int val2_idx = (int)num_siblings - 1; + + + fill_two_with_valid_data(group, val1_idx, val2_idx, 100); + + int best_idx = -1; + float *best_data = nullptr; + + //trigger a real failover + float new_best_val = 3.14159f; + float data[DataValidator::dimensions] = {new_best_val}; + //trigger a bunch of errors on the previous best sensor + unsigned val1_err_count = 0; + + for (int i = 0; i < 25; i++) { + group->put(val1_idx, timestamp, data, ++val1_err_count, 100); + group->put(val2_idx, timestamp, data, 0, 10); + } + + assert(validator1->error_count() == val1_err_count); + + //since validator1 is experiencing errors, we should see a failover to validator2 + best_data = group->get_best(timestamp + 1, &best_idx); + assert(nullptr != best_data); + assert(new_best_val == best_data[0]); + assert(best_idx == val2_idx); + //should have detected a real failover + printf("failover_count: %d \n", group->failover_count()); + assert(1 == group->failover_count()); + + //even though validator1 has encountered a bunch of errors, it hasn't failed + assert(DataValidator::ERROR_FLAG_NO_ERROR == validator1->state()); + + // although we failed over from one sensor to another, this is not the same thing tracked by failover_index + int fail_idx = group->failover_index(); + assert(-1 == fail_idx);//no failed sensor + + //since no sensor has actually hard-failed, the group failover state is NO_ERROR + assert(DataValidator::ERROR_FLAG_NO_ERROR == group->failover_state()); + + + delete group; //cleanup +} + +/** + * Verify that we get expected vibration values after injecting samples. + */ +void test_vibration() +{ + unsigned num_siblings = 0; + uint64_t timestamp = base_timestamp; + + DataValidatorGroup *group = setup_base_group(&num_siblings); + + //now we add validators + DataValidator *validator = add_validator_to_group(group); + assert(nullptr != validator); + num_siblings++; + float *vibes = validator->vibration_offset(); + assert(nullptr != vibes); + //printf("val vibes: %f \n", vibes[0]); + //should be no vibration data yet + assert(0 == vibes[0]); + + float vibe_o = group->get_vibration_offset(timestamp, 0); + //printf("group vibe_o %f \n", vibe_o); + // there should be no known vibe offset before samples are inserted + assert(-1.0f == vibe_o); + + float rms_err = 0.0f; + //insert some swinging values + insert_values_around_mean(validator, 3.14159f, 1000, &rms_err, ×tamp); + vibes = validator->vibration_offset(); + assert(nullptr != vibes); + printf("val1 vibes: %f rms_err: %f \n", vibes[0], (double)rms_err); + + vibe_o = group->get_vibration_offset(timestamp, 0); + printf("group vibe_o %f \n", vibe_o); + //the one validator's vibration offset should match the group's vibration offset + assert(vibes[0] == vibe_o); + + //this should be "The best RMS value of a non-timed out sensor" + float group_vibe_fact = group->get_vibration_factor(timestamp); + float val1_rms = (validator->rms())[0]; + printf("group_vibe_fact: %f val1_rms: %f\n", (double)group_vibe_fact, (double)val1_rms); + assert(group_vibe_fact == val1_rms); + +} + +/** + * Force once sensor to fail and ensure that we detect it + */ +void test_sensor_failure() +{ + unsigned num_siblings = 0; + uint64_t timestamp = base_timestamp; + const float sufficient_incr_value = (1.1f * 1E-6f); + const uint32_t timeout_usec = 2000;//derived from class-private value + + float val = 3.14159f; + + DataValidatorGroup *group = setup_base_group(&num_siblings); + + //now we add validators + DataValidator *validator = add_validator_to_group(group); + assert(nullptr != validator); + num_siblings++; + int val_idx = num_siblings - 1; + + fill_validator_with_samples(validator, sufficient_incr_value, &val, ×tamp); + //the best should now be the one validator we've filled with samples + + int best_idx = -1; + float *best_data = group->get_best(timestamp, &best_idx); + assert(nullptr != best_data); + //printf("best_idx: %d val_idx: %d\n", best_idx, val_idx); + assert(best_idx == val_idx); + + //now force a timeout failure in the one validator, by checking confidence long past timeout + validator->confidence(timestamp + (1.1 * timeout_usec)); + assert(DataValidator::ERROR_FLAG_TIMEOUT == (DataValidator::ERROR_FLAG_TIMEOUT & validator->state())); + + //now that the one sensor has failed, the group should detect this as well + int fail_idx = group->failover_index(); + assert(val_idx == fail_idx); + + delete group; +} + +int main(int argc, char *argv[]) +{ + (void)argc; // unused + (void)argv; // unused + + test_init(); + test_put(); + test_simple_failover(); + test_priority_switch(); + test_vibration(); + test_sensor_failure(); + + return 0; //passed +} diff --git a/validation/tests/tests_common.cpp b/validation/tests/tests_common.cpp new file mode 100644 index 0000000000..bcf18ba369 --- /dev/null +++ b/validation/tests/tests_common.cpp @@ -0,0 +1,103 @@ +/**************************************************************************** + * + * Copyright (c) 2019 Todd Stellanova. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +#include +#include "tests_common.h" + + +void insert_values_around_mean(DataValidator *validator, const float mean, uint32_t count, float *rms_err, + uint64_t *timestamp_io) +{ + uint64_t timestamp = *timestamp_io; + uint64_t timestamp_incr = 5; + const uint64_t error_count = 0; + const int priority = 50; + const float swing = 1E-2f; + double sum_dev_squares = 0.0f; + + //insert a series of values that swing around the mean + for (uint32_t i = 0; i < count; i++) { + float iter_swing = (0 == (i % 2)) ? swing : -swing; + float iter_val = mean + iter_swing; + float iter_dev = iter_val - mean; + sum_dev_squares += (iter_dev * iter_dev); + timestamp += timestamp_incr; + validator->put(timestamp, iter_val, error_count, priority); + } + + double rms = sqrt(sum_dev_squares / (double)count); + //note: this should be approximately equal to "swing" + *rms_err = (float)rms; + *timestamp_io = timestamp; +} + +void dump_validator_state(DataValidator *validator) +{ + uint32_t state = validator->state(); + printf("state: 0x%x no_data: %d stale: %d timeout:%d\n", + validator->state(), + DataValidator::ERROR_FLAG_NO_DATA & state, + DataValidator::ERROR_FLAG_STALE_DATA & state, + DataValidator::ERROR_FLAG_TIMEOUT & state + ); + validator->print(); + printf("\n"); +} + +void fill_validator_with_samples(DataValidator *validator, + const float incr_value, + float *value_io, + uint64_t *timestamp_io) +{ + uint64_t timestamp = *timestamp_io; + const uint64_t timestamp_incr = 5; //usec + const uint32_t timeout_usec = 2000;//derived from class-private value + + float val = *value_io; + const uint64_t error_count = 0; + const int priority = 50; //"medium" priority + const int equal_value_count = 100; //default is private VALUE_EQUAL_COUNT_DEFAULT + + validator->set_equal_value_threshold(equal_value_count); + validator->set_timeout(timeout_usec); + + //put a bunch of values that are all different + for (int i = 0; i < equal_value_count; i++, val += incr_value) { + timestamp += timestamp_incr; + validator->put(timestamp, val, error_count, priority); + } + + *timestamp_io = timestamp; + *value_io = val; + +} \ No newline at end of file diff --git a/validation/tests/tests_common.h b/validation/tests/tests_common.h new file mode 100644 index 0000000000..ae1351dfe1 --- /dev/null +++ b/validation/tests/tests_common.h @@ -0,0 +1,68 @@ +/**************************************************************************** + * + * Copyright (c) 2019 Todd Stellanova. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +#ifndef ECL_TESTS_COMMON_H +#define ECL_TESTS_COMMON_H + +#include + +/** + * Insert a series of samples around a mean value + * @param validator The validator under test + * @param mean The mean value + * @param count The number of samples to insert in the validator + * @param rms_err (out) calculated rms error of the inserted samples + * @param timestamp_io (in/out) in: start timestamp, out: last timestamp + */ +void insert_values_around_mean(DataValidator *validator, const float mean, uint32_t count, float *rms_err, + uint64_t *timestamp_io); + +/** + * Print out the state of a DataValidator + * @param validator + */ +void dump_validator_state(DataValidator *validator); + +/** + * Insert a time series of samples into the validator + * @param validator + * @param incr_value The amount to increment the value by on each iteration + * @param value_io (in/out) in: initial value, out: final value + * @param timestamp_io (in/out) in: initial timestamp, out: final timestamp + */ +void fill_validator_with_samples(DataValidator *validator, + const float incr_value, + float *value_io, + uint64_t *timestamp_io); + +#endif //ECL_TESTS_COMMON_H