Skip to content

Mutant Generator

pyllmut.generator.MutantGenerator

A class for generating mutants for a given Python module. This class uses an LLM to generate code mutants for specified lines in a module.

Source code in pyllmut/generator.py 
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
class MutantGenerator:
    """
    A class for generating mutants for a given Python module.
    This class uses an LLM to generate code mutants for specified lines in a module.
    """

    def __init__(
            self,
            module_content: str,
            line_number_list: Optional[List[int]] = None,
            mutants_per_line_count: int = 1,
            timeout_seconds_per_line: int = 10,
            model_type: ModelType = ModelType.GPT4oMini
    ):
        """
        Initializes the MutantGenerator.

        Args:
            module_content (str): The content of the Python module to mutate.
            line_number_list (Optional[List[int]]): List of line numbers to generate mutants for.
                If None, mutants will be generated for all code lines.
            mutants_per_line_count (int): Desired number of mutants to generate per line.
                It must be greater than 0. However, note that the LLM may not always comply
                with this number, but it usually does.
            timeout_seconds_per_line (int): Timeout in seconds for generating mutants for a line.
                It must be greater than 0.
            model_type (ModelType): The LLM model used for mutation generation. Defaults to `GPT4oMini`.

        Raises:
            AssertionError: If `mutants_per_line_count` or `timeout_seconds_per_line` is not greater than 0.
        """
        self._module_content = module_content
        self._line_number_list = line_number_list
        self._model_type = model_type

        assert (
            mutants_per_line_count > 0
        ), "mutants_per_line_count must be greater than 0"
        self._mutants_per_line_count = mutants_per_line_count

        assert (
                timeout_seconds_per_line > 0
        ), "timeout_seconds_per_line must be greater than 0"
        self._timeout_seconds_per_line = timeout_seconds_per_line

    def generate(self) -> MutationReport:
        """
        Generates mutants for the module and returns a MutationReport.

        Returns:
            MutationReport: An object containing the mutation generation results.
        """
        module_mutant_list = []
        module_timeout_info_list = []
        module_bad_response_info_list = []

        if self._line_number_list is None:
            self._line_number_list = source_manager.get_module_code_line_list(self._module_content)

        for line_number in self._line_number_list:
            logger.info(
                f"LLM is generating {self._mutants_per_line_count} mutant(s) for line {line_number}."
            )

            # `_mutants_per_line_count` serves as a guideline for the model,
            # but it is not strictly enforced.
            # The model may generate more or fewer mutants than specified.
            # It is the responsibility of PyLLMut's client to check
            # the count and take appropriate action if needed.
            (
                line_mutant_list,
                line_timeout_info,
                line_bad_response_info
            ) = self._generate_mutant_list(line_number)

            module_mutant_list += line_mutant_list

            if line_timeout_info is not None:
                module_timeout_info_list.append(line_timeout_info)

            if line_bad_response_info is not None:
                module_bad_response_info_list.append(line_bad_response_info)

        mutant_manager.classify_mutant_list(module_mutant_list)

        mutation_report = MutationReport(module_mutant_list, module_timeout_info_list, module_bad_response_info_list)

        return mutation_report

    def _generate_mutant_list(
            self,
            line_number: int
    ) -> Tuple[List[MutantInfo], Optional[PromptInfo], Optional[ResponseInfo]]:
        """
        Generates mutants for a specific line in the Python module.

        Args:
            line_number (int): The line number to generate mutants for.

        Returns:
            List[MutantInfo]: A list of MutantInfo objects generated for the specified line.
        """

        # We only generate mutants for code lines.
        if not source_manager.is_code_line(self._module_content, line_number):
            logger.info(f"Line {line_number} is not a code line, and thus, no mutants are generated for it.")
            return [], None, None

        code_line_context = source_manager.get_code_line_context(
            self._module_content, line_number
        )
        code_line = source_manager.get_code_line(self._module_content, line_number)
        prompt_content = prompt_manager.get_prompt(
            code_line_context, self._mutants_per_line_count, code_line
        )

        model = model_manager.get_model(
            self._model_type,
            self._timeout_seconds_per_line
        )

        logger.info(f"PyLLMut is using model {model}.")

        try:
            model_response = model.invoke_prompt(prompt_content)
        except openai.APITimeoutError:
            timeout_info = PromptInfo(prompt_content, self._module_content, line_number)
            return [], timeout_info, None

        try:
            mutant_dict_list = response_manager.extract_mutant_dict_list(
                model_response.get_response_content()
            )
        except ValueError:
            bad_response_info = ResponseInfo(
                prompt_content,
                self._module_content,
                line_number,
                model_response.get_sent_token_count(),
                model_response.get_response_content(),
                model_response.get_received_token_count()
            )
            return [], None, bad_response_info

        mutant_list = []
        for mutant_dict in mutant_dict_list:
            mutant = mutant_manager.get_mutant(
                prompt_content,
                model_response.get_response_content(),
                model_response.get_sent_token_count(),
                model_response.get_received_token_count(),
                self._module_content,
                line_number,
                mutant_dict,
            )
            mutant_list.append(mutant)

        return mutant_list, None, None

__init__(module_content, line_number_list=None, mutants_per_line_count=1, timeout_seconds_per_line=10, model_type=ModelType.GPT4oMini)

Initializes the MutantGenerator.

Parameters:

Name Type Description Default
module_content str

The content of the Python module to mutate.

 required
line_number_list Optional[List[int]]

List of line numbers to generate mutants for. If None, mutants will be generated for all code lines.

 None
mutants_per_line_count int

Desired number of mutants to generate per line. It must be greater than 0. However, note that the LLM may not always comply with this number, but it usually does.

 1
timeout_seconds_per_line int

Timeout in seconds for generating mutants for a line. It must be greater than 0.

 10
model_type ModelType

The LLM model used for mutation generation. Defaults to GPT4oMini.

 GPT4oMini

Raises:

Type Description
AssertionError

If mutants_per_line_count or timeout_seconds_per_line is not greater than 0.
Source code in pyllmut/generator.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
def __init__(
        self,
        module_content: str,
        line_number_list: Optional[List[int]] = None,
        mutants_per_line_count: int = 1,
        timeout_seconds_per_line: int = 10,
        model_type: ModelType = ModelType.GPT4oMini
):
    """
    Initializes the MutantGenerator.

    Args:
        module_content (str): The content of the Python module to mutate.
        line_number_list (Optional[List[int]]): List of line numbers to generate mutants for.
            If None, mutants will be generated for all code lines.
        mutants_per_line_count (int): Desired number of mutants to generate per line.
            It must be greater than 0. However, note that the LLM may not always comply
            with this number, but it usually does.
        timeout_seconds_per_line (int): Timeout in seconds for generating mutants for a line.
            It must be greater than 0.
        model_type (ModelType): The LLM model used for mutation generation. Defaults to `GPT4oMini`.

    Raises:
        AssertionError: If `mutants_per_line_count` or `timeout_seconds_per_line` is not greater than 0.
    """
    self._module_content = module_content
    self._line_number_list = line_number_list
    self._model_type = model_type

    assert (
        mutants_per_line_count > 0
    ), "mutants_per_line_count must be greater than 0"
    self._mutants_per_line_count = mutants_per_line_count

    assert (
            timeout_seconds_per_line > 0
    ), "timeout_seconds_per_line must be greater than 0"
    self._timeout_seconds_per_line = timeout_seconds_per_line

generate()

Generates mutants for the module and returns a MutationReport.

Returns:

Name Type Description
MutationReport MutationReport

An object containing the mutation generation results.
Source code in pyllmut/generator.py 
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
def generate(self) -> MutationReport:
    """
    Generates mutants for the module and returns a MutationReport.

    Returns:
        MutationReport: An object containing the mutation generation results.
    """
    module_mutant_list = []
    module_timeout_info_list = []
    module_bad_response_info_list = []

    if self._line_number_list is None:
        self._line_number_list = source_manager.get_module_code_line_list(self._module_content)

    for line_number in self._line_number_list:
        logger.info(
            f"LLM is generating {self._mutants_per_line_count} mutant(s) for line {line_number}."
        )

        # `_mutants_per_line_count` serves as a guideline for the model,
        # but it is not strictly enforced.
        # The model may generate more or fewer mutants than specified.
        # It is the responsibility of PyLLMut's client to check
        # the count and take appropriate action if needed.
        (
            line_mutant_list,
            line_timeout_info,
            line_bad_response_info
        ) = self._generate_mutant_list(line_number)

        module_mutant_list += line_mutant_list

        if line_timeout_info is not None:
            module_timeout_info_list.append(line_timeout_info)

        if line_bad_response_info is not None:
            module_bad_response_info_list.append(line_bad_response_info)

    mutant_manager.classify_mutant_list(module_mutant_list)

    mutation_report = MutationReport(module_mutant_list, module_timeout_info_list, module_bad_response_info_list)

    return mutation_report