free unncessary memory in TokenizedText

textattack/shared/scripts/run_attack_parallel.py

@@ -37,6 +37,8 @@ def attack_from_queue(args, in_queue, out_queue):
             results_gen = attack.attack_dataset([(output, text)], num_examples=1)
             result = next(results_gen)
             out_queue.put(result)
+            del output
+            del text
         except Exception as e:
             out_queue.put(e)
             exit()
@@ -103,6 +105,7 @@ def run(args):
     pbar = tqdm.tqdm(total=num_examples, smoothing=0)
     while num_results < num_examples:
         result = out_queue.get(block=True)
+
         if isinstance(result, Exception):
             raise result
         attack_log_manager.log_result(result)
@@ -119,9 +122,12 @@ def run(args):
             in_queue.put((label, text))
 
         if args.checkpoint_interval and num_results % args.checkpoint_interval == 0:
+            attack_log_manager.flush()
             checkpoint = textattack.shared.Checkpoint(args, attack_log_manager)
             checkpoint.save()
-            attack_log_manager.flush()
+        else:
+            if num_results > 0 and num_results % 50 == 0:
+                attack_log_manager.flush()
 
     pbar.close()
     print()