diff --git a/local_tests/command_line_tests.py b/local_tests/command_line_tests.py index 4dc4bf19..6edc1ff8 100644 --- a/local_tests/command_line_tests.py +++ b/local_tests/command_line_tests.py @@ -33,6 +33,7 @@ register_test('python scripts/run_attack.py --model bert-snli --recipe textfoole name='run_attack_textfooler_bert_snli_10', output_file='local_tests/outputs/run_attack_textfooler_bert_snli_10.txt', desc='Runs attack using TextFooler recipe on BERT using 10 examples from the SNLI dataset') + # # test: run_attack deepwordbug attack on 10 samples from LSTM MR # (takes about 41s) @@ -40,4 +41,20 @@ register_test('python scripts/run_attack.py --model bert-snli --recipe textfoole register_test('python scripts/run_attack.py --model lstm-mr --recipe deepwordbug --num_examples 10', name='run_attack_deepwordbug_lstm_mr_10', output_file='local_tests/outputs/run_attack_deepwordbug_lstm_mr_10.txt', - desc='Runs attack using DeepWordBUG recipe on LSTM using 10 examples from the MR dataset') \ No newline at end of file + desc='Runs attack using DeepWordBUG recipe on LSTM using 10 examples from the MR dataset') + +# +# test: run_attack targeted classification of class 2 on BERT MNLI with enable_csv +# and attack_n set, using the WordNet transformation and beam search with +# beam width 2, using language tool constraint, on 10 samples +# (takes about 171s) +# +register_test(('python scripts/run_attack.py --attack_n --goal_function targeted-classification:target_class=2 ' + '--enable_csv --model bert-mnli --num_examples 10 --transformation word-swap-wordnet ' + '--constraints lang-tool --attack beam-search:beam_width=2'), + name='run_attack_targeted2_bertmnli_wordnet_beamwidth_2_enablecsv_attackn', + output_file='local_tests/outputs/run_attack_targetedclassification2_wordnet_langtool_enable_csv_beamsearch2_attack_n_10.txt', + desc=('Runs attack using targeted classification on class 2 on BERT MNLI with' + 'enable_csv and attack_n set, using the WordNet transformation and beam ' + 'search with beam width 2, using language tool constraint, on 10 samples') + ) \ No newline at end of file diff --git a/local_tests/run_tests.py b/local_tests/run_tests.py index 53282f1a..3e49fc40 100644 --- a/local_tests/run_tests.py +++ b/local_tests/run_tests.py @@ -1,12 +1,10 @@ import os import time -from tests import tests from test_models import color_text def log_sep(): print('\n' + ('-' * 60) + '\n') - def print_gray(s): print(color_text(s, 'light_gray')) @@ -22,6 +20,8 @@ def main(): # Execute tests. start_time = time.time() passed_tests = 0 + + from tests import tests for test in tests: log_sep() test_passed = test()