How to delete test files when python unittest fails

Question

I'm using python unittest for functions that write data to JSON. I use tearDownClass to delete the output test files so they don't clutter the local repo. Ground truths are also stored as JSON files. I do want to store the output test files when tests fail, so its easier for troubleshooting.

My current implementation is to use a global boolean keep_file = False. When the unittest fails the assertion, it modifies keep_file = True. tearDownClass only deletes the files when keep_file == False. I don't like the idea of modifying global variables and the try exception blocks for each assert.

import json
import os
import unittest

from src.mymodule import foo1, foo2

# These are defined outside the class on purpose so the classmethods can access them
FILE_1 = "unittest.file1.json"
EXPECTED_FILE_1 = "expected.file1.json"

FILE_2 = "unittest.file2.json"
EXPECTED_FILE_2 = "expected.file2.json"


keep_files = False


class TestRhaPostPayload(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.get_file1()
        cls.get_file2()

    @classmethod
    def get_file1(cls):
        output1 = foo1()
        with open(FILE_1, "w") as f:
            f.write(output1)

    @classmethod
    def get_file2(cls):
        output2 = foo1()
        with open(FILE_2, "w") as f:
            f.write(output2)

    @classmethod
    def tearDownClass(cls):
        if not keep_files:
            os.remove(FILE_1)
            os.remove(FILE_2)

    def test_foo1(self):
        # code that reads in file1 and expected_file_1
        try:
            self.assert(expected_output1, output1)
        except AssertionError:
            global keep_files
            keep_files = True
            raise


    def test_foo2(self):
        # code that reads in file2 and expected_file_2
        try:
            self.assert(expected_output2, output2)
        except AssertionError:
            global keep_files
            keep_files = True
            raise

Answer 1

You could simply check, if there were any errors/failures in your test case during tear-down and only delete the files, if there were none.

How to perform this check was explained in this post.

This check is done on a TestCase instance so tearDownClass won't work. But you are using different files in different tests anyway, so you might as well use normal setUp/tearDown to remove the current file.

Here is a working example:

from pathlib import Path
from typing import Optional
from unittest import TestCase

class Test(TestCase):
    def all_tests_passed(self) -> bool:
        """Returns `True` if no errors/failures occurred at the time of calling."""
        outcome = getattr(self, "_outcome")
        if hasattr(outcome, "errors"):  # Python <=3.10
            result = self.defaultTestResult()
            getattr(self, "_feedErrorsToResult")(result, outcome.errors)
        else:  # Python >=3.11
            result = outcome.result
        return all(test != self for test, _ in result.errors + result.failures)

    def setUp(self) -> None:
        super().setUp()
        self.test_file: Optional[Path] = None

    def tearDown(self) -> None:
        super().tearDown()
        if self.test_file and self.all_tests_passed():
            self.test_file.unlink()

    def test_foo(self) -> None:
        self.test_file = Path("foo.txt")
        self.test_file.touch()
        self.assertTrue(True)

    def test_bar(self) -> None:
        self.test_file = Path("bar.txt")
        self.test_file.touch()
        self.assertTrue(False)

Running this test case leaves bar.txt in the current working directory, whereas foo.txt is gone.